From 244a4b92e832ff556491e3aeb377ff865bf4a5be Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sat, 23 Jul 2022 21:40:25 -0400 Subject: [PATCH 01/72] cosmos SDK and client upgrades (Moving Naveen's work to Azure YCSB fork) 1) SDK upgrade to 4.28.0 2) Update operation with Patch 3) removed Retry 4) updated error log 5) Adding status code for errors 6)Adding diagnosticsLatencyThresholdInMS --- .../java/site/ycsb/db/AzureCosmosClient.java | 149 ++++++++++-------- pom.xml | 2 +- 2 files changed, 86 insertions(+), 65 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 040485d301..88e013f6f7 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -16,21 +16,6 @@ package site.ycsb.db; -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.Vector; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.CosmosClient; import com.azure.cosmos.CosmosClientBuilder; @@ -42,6 +27,7 @@ import com.azure.cosmos.ThrottlingRetryOptions; import com.azure.cosmos.models.CosmosItemRequestOptions; import com.azure.cosmos.models.CosmosItemResponse; +import com.azure.cosmos.models.CosmosPatchOperations; import com.azure.cosmos.models.CosmosQueryRequestOptions; import com.azure.cosmos.models.FeedResponse; import com.azure.cosmos.models.PartitionKey; @@ -51,15 +37,28 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import site.ycsb.ByteIterator; import site.ycsb.DB; import site.ycsb.DBException; import site.ycsb.Status; import site.ycsb.StringByteIterator; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.Vector; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + /** - * Azure Cosmos DB Java SDK 4.6.0 client for YCSB. + * Azure Cosmos DB Java SDK 4.28.0 client for YCSB. */ public class AzureCosmosClient extends DB { @@ -74,7 +73,7 @@ public class AzureCosmosClient extends DB { private static final int DEFAULT_MAX_DEGREE_OF_PARALLELISM = -1; private static final int DEFAULT_MAX_BUFFERED_ITEM_COUNT = 0; private static final int DEFAULT_PREFERRED_PAGE_SIZE = -1; - public static final int NUM_UPDATE_ATTEMPTS = 4; + private static final int DEFAULT_DIAGNOSTICS_LATENCY_THRESHOLD_IN_MS = -1; private static final boolean DEFAULT_INCLUDE_EXCEPTION_STACK_IN_LOG = false; private static final String DEFAULT_USER_AGENT = "azurecosmos-ycsb"; @@ -93,6 +92,7 @@ public class AzureCosmosClient extends DB { private static int maxDegreeOfParallelism; private static int maxBufferedItemCount; private static int preferredPageSize; + private static int diagnosticsLatencyThresholdInMS; private static boolean includeExceptionStackInLog; private static Map containerCache; private static String userAgent; @@ -141,6 +141,10 @@ private void initAzureCosmosClient() throws DBException { AzureCosmosClient.preferredPageSize = this.getIntProperty("azurecosmos.preferredPageSize", DEFAULT_PREFERRED_PAGE_SIZE); + AzureCosmosClient.diagnosticsLatencyThresholdInMS = this.getIntProperty( + "azurecosmos.diagnosticsLatencyThresholdInMS", + DEFAULT_DIAGNOSTICS_LATENCY_THRESHOLD_IN_MS); + AzureCosmosClient.includeExceptionStackInLog = this.getBooleanProperty("azurecosmos.includeExceptionStackInLog", DEFAULT_INCLUDE_EXCEPTION_STACK_IN_LOG); @@ -310,10 +314,20 @@ public Status read(String table, String key, Set fields, Map 0 && + response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { + LOGGER.warn(response.getDiagnostics().toString()); + } + return Status.OK; } catch (CosmosException e) { - LOGGER.error("Failed to read key {} in collection {} in database {}", key, table, AzureCosmosClient.databaseName, - e); + int statusCode = e.getStatusCode(); + if (!AzureCosmosClient.includeExceptionStackInLog) { + e = null; + } + LOGGER.error("Failed to read key {} in collection {} in database {} statusCode {}", key, table, + AzureCosmosClient.databaseName, statusCode, e); return Status.NOT_FOUND; } } @@ -333,7 +347,7 @@ public Status read(String table, String key, Set fields, Map fields, - Vector> result) { + Vector> result) { try { CosmosQueryRequestOptions queryOptions = new CosmosQueryRequestOptions(); queryOptions.setMaxDegreeOfParallelism(AzureCosmosClient.maxDegreeOfParallelism); @@ -369,11 +383,12 @@ public Status scan(String table, String startkey, int recordcount, Set f } return Status.OK; } catch (CosmosException e) { + int statusCode = e.getStatusCode(); if (!AzureCosmosClient.includeExceptionStackInLog) { e = null; } - LOGGER.error("Failed to query key {} from collection {} in database {}", startkey, table, - AzureCosmosClient.databaseName, e); + LOGGER.error("Failed to query key {} from collection {} in database {} statusCode {}", startkey, table, + AzureCosmosClient.databaseName, statusCode, e); } return Status.ERROR; } @@ -390,43 +405,33 @@ public Status scan(String table, String startkey, int recordcount, Set f */ @Override public Status update(String table, String key, Map values) { + try { + CosmosContainer container = AzureCosmosClient.containerCache.get(table); + if (container == null) { + container = AzureCosmosClient.database.getContainer(table); + AzureCosmosClient.containerCache.put(table, container); + } - String readEtag = ""; - - // Azure Cosmos DB does not have patch support. Until then, we need to read - // the document, update it, and then write it back. - // This could be made more efficient by using a stored procedure - // and doing the read/modify write on the server side. Perhaps - // that will be a future improvement. - for (int attempt = 0; attempt < NUM_UPDATE_ATTEMPTS; attempt++) { - try { - CosmosContainer container = AzureCosmosClient.containerCache.get(table); - if (container == null) { - container = AzureCosmosClient.database.getContainer(table); - AzureCosmosClient.containerCache.put(table, container); - } - - CosmosItemResponse response = container.readItem(key, new PartitionKey(key), ObjectNode.class); - readEtag = response.getETag(); - ObjectNode node = response.getItem(); - - for (Entry pair : values.entrySet()) { - node.put(pair.getKey(), pair.getValue().toString()); - } + CosmosPatchOperations cosmosPatchOperations = CosmosPatchOperations.create(); + for (Entry pair : values.entrySet()) { + cosmosPatchOperations.replace("/" + pair.getKey(), pair.getValue().toString()); + } - CosmosItemRequestOptions requestOptions = new CosmosItemRequestOptions(); - requestOptions.setIfMatchETag(readEtag); - PartitionKey pk = new PartitionKey(key); - container.replaceItem(node, key, pk, requestOptions); + PartitionKey pk = new PartitionKey(key); + CosmosItemResponse response = container.patchItem(key, pk, cosmosPatchOperations, ObjectNode.class); + if (diagnosticsLatencyThresholdInMS > 0 && + response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { + LOGGER.warn(response.getDiagnostics().toString()); + } - return Status.OK; - } catch (CosmosException e) { - if (!AzureCosmosClient.includeExceptionStackInLog) { - e = null; - } - LOGGER.error("Failed to update key {} to collection {} in database {} on attempt {}", key, table, - AzureCosmosClient.databaseName, attempt, e); + return Status.OK; + } catch (CosmosException e) { + int statusCode = e.getStatusCode(); + if (!AzureCosmosClient.includeExceptionStackInLog) { + e = null; } + LOGGER.error("Failed to update key {} to collection {} in database {} statusCode {}", key, table, + AzureCosmosClient.databaseName, statusCode, e); } return Status.ERROR; @@ -460,18 +465,26 @@ public Status insert(String table, String key, Map values) for (Map.Entry pair : values.entrySet()) { node.put(pair.getKey(), pair.getValue().toString()); } + CosmosItemResponse response; if (AzureCosmosClient.useUpsert) { - container.upsertItem(node, pk, new CosmosItemRequestOptions()); + response = container.upsertItem(node, pk, new CosmosItemRequestOptions()); } else { - container.createItem(node, pk, new CosmosItemRequestOptions()); + response = container.createItem(node, pk, new CosmosItemRequestOptions()); + } + + if (diagnosticsLatencyThresholdInMS > 0 && + response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { + LOGGER.warn(response.getDiagnostics().toString()); } + return Status.OK; } catch (CosmosException e) { + int statusCode = e.getStatusCode(); if (!AzureCosmosClient.includeExceptionStackInLog) { e = null; } - LOGGER.error("Failed to insert key {} to collection {} in database {}", key, table, - AzureCosmosClient.databaseName, e); + LOGGER.error("Failed to insert key {} to collection {} in database {} statusCode {}", key, table, + AzureCosmosClient.databaseName, statusCode, e); } return Status.ERROR; } @@ -487,14 +500,22 @@ public Status delete(String table, String key) { container = AzureCosmosClient.database.getContainer(table); AzureCosmosClient.containerCache.put(table, container); } - container.deleteItem(key, new PartitionKey(key), new CosmosItemRequestOptions()); + CosmosItemResponse response = container.deleteItem(key, + new PartitionKey(key), + new CosmosItemRequestOptions()); + if (diagnosticsLatencyThresholdInMS > 0 && + response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { + LOGGER.warn(response.getDiagnostics().toString()); + } return Status.OK; - } catch (Exception e) { + } catch (CosmosException e) { + int statusCode = e.getStatusCode(); if (!AzureCosmosClient.includeExceptionStackInLog) { e = null; } - LOGGER.error("Failed to delete key {} in collection {}", key, table, e); + LOGGER.error("Failed to delete key {} in collection {} database {} statusCode {}", key, table, + AzureCosmosClient.databaseName, statusCode, e); } return Status.ERROR; } @@ -514,4 +535,4 @@ private String createSelectTop(Set fields, int top) { return result.toString(); } } -} +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index de2d5eea23..28e8760b80 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ LICENSE file. 3.1.2 4.4.1 1.8.2 - 4.8.0 + 4.28.0 4.0.0 3.0.0 2.0.1 From d5ef9ba3755289ec60786afe5d19e8e4b2a0c83e Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sun, 24 Jul 2022 16:36:35 -0400 Subject: [PATCH 02/72] Update azurecosmos.properties adding request diagnostics configuration --- azurecosmos/conf/azurecosmos.properties | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/azurecosmos/conf/azurecosmos.properties b/azurecosmos/conf/azurecosmos.properties index ac2a6c69e9..389ba7ba93 100644 --- a/azurecosmos/conf/azurecosmos.properties +++ b/azurecosmos/conf/azurecosmos.properties @@ -16,8 +16,8 @@ # See https://docs.microsoft.com/en-us/azure/cosmos-db/performance-tips-java-sdk-v4-sql for details on some of the options below. # Azure Cosmos DB host uri (ex: https://p3rf.documents.azure.com:443/) and primary key. -# azurecosmos.primaryKey = -# azurecosmos.uri = +# azurecosmos.primaryKey = +# azurecosmos.uri = # Database to be used, if not specified 'ycsb' will be used. # azurecosmos.databaseName = ycsb @@ -31,6 +31,10 @@ # The default is false to reduce output size. # azurecosmos.includeExceptionStackInLog = false +# Determines if full request diagnostics need to be printed for high latency requests. +# The default is -1(no diagnostics at all)to reduce output size. +# azurecosmos.diagnosticsLatencyThresholdInMS = -1 + # The value to be appended to the user-agent header. # In most cases, you should leave this as "azurecosmos-ycsb". # azurecosmos.userAgent = azurecosmos-ycsb @@ -54,7 +58,7 @@ # Set the maximum retry duration in seconds. # azurecosmos.maxRetryWaitTimeInSeconds = 30 -# Set the value of the connection pool size in gateway mode. +# Set the value of the connection pool size in gateway mode. # azurecosmos.gatewayMaxConnectionPoolSize = 30 # Set the value of the max connections per endpoint in direct mode. @@ -82,3 +86,6 @@ # Sets the preferred page size when scanning. # Default value is -1. # azurecosmos.preferredPageSize = -1 + +# output file location +# exportfile = \ No newline at end of file From 11507aa1ec01d92920f4713cc8225e27947dce1e Mon Sep 17 00:00:00 2001 From: Microsoft Open Source Date: Mon, 25 Jul 2022 10:33:00 -0700 Subject: [PATCH 03/72] CODE_OF_CONDUCT.md committed --- CODE_OF_CONDUCT.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..f9ba8cf65f --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,9 @@ +# Microsoft Open Source Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). + +Resources: + +- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) +- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns From 6849cc75a0bc046f3c60182345d14c854d113b44 Mon Sep 17 00:00:00 2001 From: Microsoft Open Source Date: Mon, 25 Jul 2022 10:33:01 -0700 Subject: [PATCH 04/72] LICENSE committed --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..9e841e7a26 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE From ed2b8f7d8bec1fa7bfc89ae320f11e561f213ee8 Mon Sep 17 00:00:00 2001 From: Microsoft Open Source Date: Mon, 25 Jul 2022 10:33:02 -0700 Subject: [PATCH 05/72] README.md updated to template --- README.md | 95 ++++++++++++++----------------------------------------- 1 file changed, 24 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 84fca3011d..5cd7cecfc8 100644 --- a/README.md +++ b/README.md @@ -1,80 +1,33 @@ - +- Improving this README.MD file to provide a great experience +- Updating SUPPORT.MD with content about this project's support experience +- Understanding the security reporting process in SECURITY.MD +- Remove this section from the README -YCSB -==================================== -[![Build Status](https://travis-ci.org/brianfrankcooper/YCSB.png?branch=master)](https://travis-ci.org/brianfrankcooper/YCSB) +## Contributing +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. +When you submit a pull request, a CLA bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. -Links ------ -* To get here, use https://ycsb.site -* [Our project docs](https://github.com/brianfrankcooper/YCSB/wiki) -* [The original announcement from Yahoo!](https://labs.yahoo.com/news/yahoo-cloud-serving-benchmark/) +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. -Getting Started ---------------- +## Trademarks -1. Download the [latest release of YCSB](https://github.com/brianfrankcooper/YCSB/releases/latest): - - ```sh - curl -O --location https://github.com/brianfrankcooper/YCSB/releases/download/0.17.0/ycsb-0.17.0.tar.gz - tar xfvz ycsb-0.17.0.tar.gz - cd ycsb-0.17.0 - ``` - -2. Set up a database to benchmark. There is a README file under each binding - directory. - -3. Run YCSB command. - - On Linux: - ```sh - bin/ycsb.sh load basic -P workloads/workloada - bin/ycsb.sh run basic -P workloads/workloada - ``` - - On Windows: - ```bat - bin/ycsb.bat load basic -P workloads\workloada - bin/ycsb.bat run basic -P workloads\workloada - ``` - - Running the `ycsb` command without any argument will print the usage. - - See https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload - for a detailed documentation on how to run a workload. - - See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties for - the list of available workload properties. - - -Building from source --------------------- - -YCSB requires the use of Maven 3; if you use Maven 2, you may see [errors -such as these](https://github.com/brianfrankcooper/YCSB/issues/406). - -To build the full distribution, with all database bindings: - - mvn clean package - -To build a single database binding: - - mvn -pl site.ycsb:mongodb-binding -am clean package +This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft +trademarks or logos is subject to and must follow +[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). +Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. +Any use of third-party trademarks or logos are subject to those third-party's policies. From a0d371937e04200cd010bab42c455a769960d57b Mon Sep 17 00:00:00 2001 From: Microsoft Open Source Date: Mon, 25 Jul 2022 10:33:03 -0700 Subject: [PATCH 06/72] SECURITY.md committed --- SECURITY.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..869fdfe2b2 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). + + From 9a2d92b5c380218f97893a5c6f22b3444ddcc317 Mon Sep 17 00:00:00 2001 From: Microsoft Open Source Date: Mon, 25 Jul 2022 10:33:04 -0700 Subject: [PATCH 07/72] SUPPORT.md committed --- SUPPORT.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 SUPPORT.md diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000000..291d4d4373 --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,25 @@ +# TODO: The maintainer of this repo has not yet edited this file + +**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? + +- **No CSS support:** Fill out this template with information about how to file issues and get help. +- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. +- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. + +*Then remove this first heading from this SUPPORT.MD file before publishing your repo.* + +# Support + +## How to file issues and get help + +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or +feature request as a new Issue. + +For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE +FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER +CHANNEL. WHERE WILL YOU HELP PEOPLE?**. + +## Microsoft Support Policy + +Support for this **PROJECT or PRODUCT** is limited to the resources listed above. From 8a46ed68cf1f2fad6ce8d43447e93b37be955e53 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sat, 30 Jul 2022 22:20:16 -0400 Subject: [PATCH 08/72] log4j2 setup --- azurecosmos/pom.xml | 35 ++++++++++--------- .../src/main/resources/log4j.properties | 30 ---------------- azurecosmos/src/main/resources/log4j2.xml | 27 ++++++++++++++ 3 files changed, 45 insertions(+), 47 deletions(-) delete mode 100644 azurecosmos/src/main/resources/log4j.properties create mode 100644 azurecosmos/src/main/resources/log4j2.xml diff --git a/azurecosmos/pom.xml b/azurecosmos/pom.xml index 1b45895fa7..b1c2a28ed9 100644 --- a/azurecosmos/pom.xml +++ b/azurecosmos/pom.xml @@ -30,27 +30,28 @@ LICENSE file. jar + com.azure azure-cosmos ${azurecosmos.version} - - - org.slf4j - slf4j-api - 1.7.5 - - - org.slf4j - slf4j-log4j12 - 1.7.5 - - - log4j - log4j - 1.2.17 - - + + + org.slf4j + slf4j-api + 1.7.5 + + + org.slf4j + slf4j-log4j12 + 1.7.5 + + + log4j + log4j + 1.2.17 + + site.ycsb core ${project.version} diff --git a/azurecosmos/src/main/resources/log4j.properties b/azurecosmos/src/main/resources/log4j.properties deleted file mode 100644 index 49a3f1a7d2..0000000000 --- a/azurecosmos/src/main/resources/log4j.properties +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2018 YCSB contributors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -#define the console appender -log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender - -# now define the layout for the appender -log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout -log4j.appender.consoleAppender.layout.ConversionPattern=%-4r [%t] %-5p %c %x -%m%n - -# now map our console appender as a root logger, means all log messages will go -# to this appender -log4j.rootLogger = INFO, consoleAppender - -# Set HTTP components' logger to INFO -log4j.category.org.apache.http=INFO -log4j.category.org.apache.http.wire=INFO -log4j.category.org.apache.http.headers=INFO diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..314490e888 --- /dev/null +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -0,0 +1,27 @@ + + + + %-4r [%t] %-5p %c %x -%m%n + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 4cfdf4989343ecf9fa4b2a85cb591945a0781ce7 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sat, 30 Jul 2022 22:29:41 -0400 Subject: [PATCH 09/72] Update pom.xml --- azurecosmos/pom.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azurecosmos/pom.xml b/azurecosmos/pom.xml index b1c2a28ed9..46ce4dd3f6 100644 --- a/azurecosmos/pom.xml +++ b/azurecosmos/pom.xml @@ -37,19 +37,19 @@ LICENSE file. ${azurecosmos.version} - org.slf4j - slf4j-api - 1.7.5 + org.apache.logging.log4j + log4j-api + 2.7 - org.slf4j - slf4j-log4j12 - 1.7.5 + org.apache.logging.log4j + log4j-core + 2.7 - log4j - log4j - 1.2.17 + org.apache.logging.log4j + log4j-slf4j-impl + 2.7 site.ycsb From e73375aaca834d056f399abe262a65a993f99a84 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sat, 30 Jul 2022 23:37:25 -0400 Subject: [PATCH 10/72] logging --- .../java/site/ycsb/db/AzureCosmosClient.java | 517 ------------------ azurecosmos/src/main/resources/log4j2.xml | 6 +- 2 files changed, 5 insertions(+), 518 deletions(-) delete mode 100644 azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java deleted file mode 100644 index 040485d301..0000000000 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ /dev/null @@ -1,517 +0,0 @@ -/* - * Copyright (c) 2018 YCSB contributors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. See accompanying LICENSE file. - */ - -package site.ycsb.db; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.Vector; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.azure.cosmos.ConsistencyLevel; -import com.azure.cosmos.CosmosClient; -import com.azure.cosmos.CosmosClientBuilder; -import com.azure.cosmos.CosmosContainer; -import com.azure.cosmos.CosmosDatabase; -import com.azure.cosmos.CosmosException; -import com.azure.cosmos.DirectConnectionConfig; -import com.azure.cosmos.GatewayConnectionConfig; -import com.azure.cosmos.ThrottlingRetryOptions; -import com.azure.cosmos.models.CosmosItemRequestOptions; -import com.azure.cosmos.models.CosmosItemResponse; -import com.azure.cosmos.models.CosmosQueryRequestOptions; -import com.azure.cosmos.models.FeedResponse; -import com.azure.cosmos.models.PartitionKey; -import com.azure.cosmos.models.SqlParameter; -import com.azure.cosmos.models.SqlQuerySpec; -import com.azure.cosmos.util.CosmosPagedIterable; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; - -import site.ycsb.ByteIterator; -import site.ycsb.DB; -import site.ycsb.DBException; -import site.ycsb.Status; -import site.ycsb.StringByteIterator; - -/** - * Azure Cosmos DB Java SDK 4.6.0 client for YCSB. - */ - -public class AzureCosmosClient extends DB { - - protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - // Default configuration values - private static final ConsistencyLevel DEFAULT_CONSISTENCY_LEVEL = ConsistencyLevel.SESSION; - private static final String DEFAULT_DATABASE_NAME = "ycsb"; - private static final boolean DEFAULT_USE_GATEWAY = false; - private static final boolean DEFAULT_USE_UPSERT = false; - private static final int DEFAULT_MAX_DEGREE_OF_PARALLELISM = -1; - private static final int DEFAULT_MAX_BUFFERED_ITEM_COUNT = 0; - private static final int DEFAULT_PREFERRED_PAGE_SIZE = -1; - public static final int NUM_UPDATE_ATTEMPTS = 4; - private static final boolean DEFAULT_INCLUDE_EXCEPTION_STACK_IN_LOG = false; - private static final String DEFAULT_USER_AGENT = "azurecosmos-ycsb"; - - private static final Logger LOGGER = LoggerFactory.getLogger(AzureCosmosClient.class); - - /** - * Count the number of times initialized to teardown on the last - * {@link #cleanup()}. - */ - private static final AtomicInteger INIT_COUNT = new AtomicInteger(0); - - private static CosmosClient client; - private static CosmosDatabase database; - private static String databaseName; - private static boolean useUpsert; - private static int maxDegreeOfParallelism; - private static int maxBufferedItemCount; - private static int preferredPageSize; - private static boolean includeExceptionStackInLog; - private static Map containerCache; - private static String userAgent; - - @Override - public void init() throws DBException { - INIT_COUNT.incrementAndGet(); - - synchronized (INIT_COUNT) { - if (client != null) { - return; - } - try { - initAzureCosmosClient(); - } catch (Exception e) { - throw new DBException(e); - } - } - } - - private void initAzureCosmosClient() throws DBException { - - // Connection properties - String primaryKey = this.getStringProperty("azurecosmos.primaryKey", null); - if (primaryKey == null || primaryKey.isEmpty()) { - throw new DBException("Missing primary key required to connect to the database."); - } - - String uri = this.getStringProperty("azurecosmos.uri", null); - if (primaryKey == null || primaryKey.isEmpty()) { - throw new DBException("Missing uri required to connect to the database."); - } - - AzureCosmosClient.userAgent = this.getStringProperty("azurecosmos.userAgent", DEFAULT_USER_AGENT); - - AzureCosmosClient.useUpsert = this.getBooleanProperty("azurecosmos.useUpsert", DEFAULT_USE_UPSERT); - - AzureCosmosClient.databaseName = this.getStringProperty("azurecosmos.databaseName", DEFAULT_DATABASE_NAME); - - AzureCosmosClient.maxDegreeOfParallelism = this.getIntProperty("azurecosmos.maxDegreeOfParallelism", - DEFAULT_MAX_DEGREE_OF_PARALLELISM); - - AzureCosmosClient.maxBufferedItemCount = this.getIntProperty("azurecosmos.maxBufferedItemCount", - DEFAULT_MAX_BUFFERED_ITEM_COUNT); - - AzureCosmosClient.preferredPageSize = this.getIntProperty("azurecosmos.preferredPageSize", - DEFAULT_PREFERRED_PAGE_SIZE); - - AzureCosmosClient.includeExceptionStackInLog = this.getBooleanProperty("azurecosmos.includeExceptionStackInLog", - DEFAULT_INCLUDE_EXCEPTION_STACK_IN_LOG); - - ConsistencyLevel consistencyLevel = ConsistencyLevel.valueOf( - this.getStringProperty("azurecosmos.consistencyLevel", DEFAULT_CONSISTENCY_LEVEL.toString().toUpperCase())); - boolean useGateway = this.getBooleanProperty("azurecosmos.useGateway", DEFAULT_USE_GATEWAY); - - ThrottlingRetryOptions retryOptions = new ThrottlingRetryOptions(); - int maxRetryAttemptsOnThrottledRequests = this.getIntProperty("azurecosmos.maxRetryAttemptsOnThrottledRequests", - -1); - if (maxRetryAttemptsOnThrottledRequests != -1) { - retryOptions.setMaxRetryAttemptsOnThrottledRequests(maxRetryAttemptsOnThrottledRequests); - } - - // Direct connection config options. - DirectConnectionConfig directConnectionConfig = new DirectConnectionConfig(); - int directMaxConnectionsPerEndpoint = this.getIntProperty("azurecosmos.directMaxConnectionsPerEndpoint", -1); - if (directMaxConnectionsPerEndpoint != -1) { - directConnectionConfig.setMaxConnectionsPerEndpoint(directMaxConnectionsPerEndpoint); - } - - int directIdleConnectionTimeoutInSeconds = this.getIntProperty("azurecosmos.directIdleConnectionTimeoutInSeconds", - -1); - if (directIdleConnectionTimeoutInSeconds != -1) { - directConnectionConfig.setIdleConnectionTimeout(Duration.ofSeconds(directIdleConnectionTimeoutInSeconds)); - } - - // Gateway connection config options. - GatewayConnectionConfig gatewayConnectionConfig = new GatewayConnectionConfig(); - - int gatewayMaxConnectionPoolSize = this.getIntProperty("azurecosmos.gatewayMaxConnectionPoolSize", -1); - if (gatewayMaxConnectionPoolSize != -1) { - gatewayConnectionConfig.setMaxConnectionPoolSize(gatewayMaxConnectionPoolSize); - } - - int gatewayIdleConnectionTimeoutInSeconds = this.getIntProperty("azurecosmos.gatewayIdleConnectionTimeoutInSeconds", - -1); - if (gatewayIdleConnectionTimeoutInSeconds != -1) { - gatewayConnectionConfig.setIdleConnectionTimeout(Duration.ofSeconds(gatewayIdleConnectionTimeoutInSeconds)); - } - - try { - LOGGER.info( - "Creating Cosmos DB client {}, useGateway={}, consistencyLevel={}," - + " maxRetryAttemptsOnThrottledRequests={}, maxRetryWaitTimeInSeconds={}" - + " useUpsert={}, maxDegreeOfParallelism={}, maxBufferedItemCount={}, preferredPageSize={}", - uri, useGateway, consistencyLevel.toString(), retryOptions.getMaxRetryAttemptsOnThrottledRequests(), - retryOptions.getMaxRetryWaitTime().toMillis() / 1000, AzureCosmosClient.useUpsert, - AzureCosmosClient.maxDegreeOfParallelism, AzureCosmosClient.maxBufferedItemCount, - AzureCosmosClient.preferredPageSize); - - CosmosClientBuilder builder = new CosmosClientBuilder().endpoint(uri).key(primaryKey) - .throttlingRetryOptions(retryOptions).consistencyLevel(consistencyLevel).userAgentSuffix(userAgent); - - if (useGateway) { - builder = builder.gatewayMode(gatewayConnectionConfig); - } else { - builder = builder.directMode(directConnectionConfig); - } - - AzureCosmosClient.client = builder.buildClient(); - LOGGER.info("Azure Cosmos DB connection created to {}", uri); - } catch (IllegalArgumentException e) { - if (!AzureCosmosClient.includeExceptionStackInLog) { - e = null; - } - throw new DBException("Illegal argument passed in. Check the format of your parameters.", e); - } - - AzureCosmosClient.containerCache = new ConcurrentHashMap<>(); - - // Verify the database exists - try { - AzureCosmosClient.database = AzureCosmosClient.client.getDatabase(databaseName); - AzureCosmosClient.database.read(); - } catch (CosmosException e) { - if (!AzureCosmosClient.includeExceptionStackInLog) { - e = null; - } - throw new DBException( - "Invalid database name (" + AzureCosmosClient.databaseName + ") or failed to read database.", e); - } - } - - private String getStringProperty(String propertyName, String defaultValue) { - return getProperties().getProperty(propertyName, defaultValue); - } - - private boolean getBooleanProperty(String propertyName, boolean defaultValue) { - String stringVal = getProperties().getProperty(propertyName, null); - if (stringVal == null) { - return defaultValue; - } - return Boolean.parseBoolean(stringVal); - } - - private int getIntProperty(String propertyName, int defaultValue) { - String stringVal = getProperties().getProperty(propertyName, null); - if (stringVal == null) { - return defaultValue; - } - try { - return Integer.parseInt(stringVal); - } catch (NumberFormatException e) { - return defaultValue; - } - } - - /** - * Cleanup any state for this DB. Called once per DB instance; there is one DB - * instance per client thread. - */ - @Override - public void cleanup() throws DBException { - synchronized (INIT_COUNT) { - if (INIT_COUNT.decrementAndGet() <= 0 && AzureCosmosClient.client != null) { - try { - AzureCosmosClient.client.close(); - } catch (Exception e) { - if (!AzureCosmosClient.includeExceptionStackInLog) { - e = null; - } - LOGGER.error("Could not close DocumentClient", e); - } finally { - AzureCosmosClient.client = null; - } - } - } - } - - /** - * Read a record from the database. Each field/value pair from the result will - * be stored in a HashMap. - * - * @param table The name of the table - * @param key The record key of the record to read. - * @param fields The list of fields to read, or null for all of them - * @param result A HashMap of field/value pairs for the result - * @return Zero on success, a non-zero error code on error - */ - @Override - public Status read(String table, String key, Set fields, Map result) { - try { - CosmosContainer container = AzureCosmosClient.containerCache.get(table); - if (container == null) { - container = AzureCosmosClient.database.getContainer(table); - AzureCosmosClient.containerCache.put(table, container); - } - - CosmosItemResponse response = container.readItem(key, new PartitionKey(key), ObjectNode.class); - ObjectNode node = response.getItem(); - Map stringResults = new HashMap<>(node.size()); - if (fields == null) { - Iterator> iter = node.fields(); - while (iter.hasNext()) { - Entry pair = iter.next(); - stringResults.put(pair.getKey().toString(), pair.getValue().toString()); - } - StringByteIterator.putAllAsByteIterators(result, stringResults); - } else { - Iterator> iter = node.fields(); - while (iter.hasNext()) { - Entry pair = iter.next(); - if (fields.contains(pair.getKey())) { - stringResults.put(pair.getKey().toString(), pair.getValue().toString()); - } - } - StringByteIterator.putAllAsByteIterators(result, stringResults); - } - return Status.OK; - } catch (CosmosException e) { - LOGGER.error("Failed to read key {} in collection {} in database {}", key, table, AzureCosmosClient.databaseName, - e); - return Status.NOT_FOUND; - } - } - - /** - * Perform a range scan for a set of records in the database. Each field/value - * pair from the result will be stored in a HashMap. - * - * - * @param table The name of the table - * @param startkey The record key of the first record to read. - * @param recordcount The number of records to read - * @param fields The list of fields to read, or null for all of them - * @param result A Vector of HashMaps, where each HashMap is a set - * field/value pairs for one record - * @return Zero on success, a non-zero error code on error - */ - @Override - public Status scan(String table, String startkey, int recordcount, Set fields, - Vector> result) { - try { - CosmosQueryRequestOptions queryOptions = new CosmosQueryRequestOptions(); - queryOptions.setMaxDegreeOfParallelism(AzureCosmosClient.maxDegreeOfParallelism); - queryOptions.setMaxBufferedItemCount(AzureCosmosClient.maxBufferedItemCount); - - CosmosContainer container = AzureCosmosClient.containerCache.get(table); - if (container == null) { - container = AzureCosmosClient.database.getContainer(table); - AzureCosmosClient.containerCache.put(table, container); - } - - List paramList = new ArrayList<>(); - paramList.add(new SqlParameter("@startkey", startkey)); - - SqlQuerySpec querySpec = new SqlQuerySpec( - this.createSelectTop(fields, recordcount) + " FROM root r WHERE r.id >= @startkey", paramList); - CosmosPagedIterable pagedIterable = container.queryItems(querySpec, queryOptions, ObjectNode.class); - Iterator> pageIterator = pagedIterable - .iterableByPage(AzureCosmosClient.preferredPageSize).iterator(); - while (pageIterator.hasNext()) { - List pageDocs = pageIterator.next().getResults(); - for (ObjectNode doc : pageDocs) { - Map stringResults = new HashMap<>(doc.size()); - Iterator> nodeIterator = doc.fields(); - while (nodeIterator.hasNext()) { - Entry pair = nodeIterator.next(); - stringResults.put(pair.getKey().toString(), pair.getValue().toString()); - } - HashMap byteResults = new HashMap<>(doc.size()); - StringByteIterator.putAllAsByteIterators(byteResults, stringResults); - result.add(byteResults); - } - } - return Status.OK; - } catch (CosmosException e) { - if (!AzureCosmosClient.includeExceptionStackInLog) { - e = null; - } - LOGGER.error("Failed to query key {} from collection {} in database {}", startkey, table, - AzureCosmosClient.databaseName, e); - } - return Status.ERROR; - } - - /** - * Update a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record key, - * overwriting any existing values with the same field name. - * - * @param table The name of the table - * @param key The record key of the record to write. - * @param values A HashMap of field/value pairs to update in the record - * @return Zero on success, a non-zero error code on error - */ - @Override - public Status update(String table, String key, Map values) { - - String readEtag = ""; - - // Azure Cosmos DB does not have patch support. Until then, we need to read - // the document, update it, and then write it back. - // This could be made more efficient by using a stored procedure - // and doing the read/modify write on the server side. Perhaps - // that will be a future improvement. - for (int attempt = 0; attempt < NUM_UPDATE_ATTEMPTS; attempt++) { - try { - CosmosContainer container = AzureCosmosClient.containerCache.get(table); - if (container == null) { - container = AzureCosmosClient.database.getContainer(table); - AzureCosmosClient.containerCache.put(table, container); - } - - CosmosItemResponse response = container.readItem(key, new PartitionKey(key), ObjectNode.class); - readEtag = response.getETag(); - ObjectNode node = response.getItem(); - - for (Entry pair : values.entrySet()) { - node.put(pair.getKey(), pair.getValue().toString()); - } - - CosmosItemRequestOptions requestOptions = new CosmosItemRequestOptions(); - requestOptions.setIfMatchETag(readEtag); - PartitionKey pk = new PartitionKey(key); - container.replaceItem(node, key, pk, requestOptions); - - return Status.OK; - } catch (CosmosException e) { - if (!AzureCosmosClient.includeExceptionStackInLog) { - e = null; - } - LOGGER.error("Failed to update key {} to collection {} in database {} on attempt {}", key, table, - AzureCosmosClient.databaseName, attempt, e); - } - } - - return Status.ERROR; - } - - /** - * Insert a record in the database. Any field/value pairs in the specified - * values HashMap will be written into the record with the specified record key. - * - * @param table The name of the table - * @param key The record key of the record to insert. - * @param values A HashMap of field/value pairs to insert in the record - * @return Zero on success, a non-zero error code on error - */ - @Override - public Status insert(String table, String key, Map values) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Insert key: {} into table: {}", key, table); - } - - try { - CosmosContainer container = AzureCosmosClient.containerCache.get(table); - if (container == null) { - container = AzureCosmosClient.database.getContainer(table); - AzureCosmosClient.containerCache.put(table, container); - } - PartitionKey pk = new PartitionKey(key); - ObjectNode node = OBJECT_MAPPER.createObjectNode(); - - node.put("id", key); - for (Map.Entry pair : values.entrySet()) { - node.put(pair.getKey(), pair.getValue().toString()); - } - if (AzureCosmosClient.useUpsert) { - container.upsertItem(node, pk, new CosmosItemRequestOptions()); - } else { - container.createItem(node, pk, new CosmosItemRequestOptions()); - } - return Status.OK; - } catch (CosmosException e) { - if (!AzureCosmosClient.includeExceptionStackInLog) { - e = null; - } - LOGGER.error("Failed to insert key {} to collection {} in database {}", key, table, - AzureCosmosClient.databaseName, e); - } - return Status.ERROR; - } - - @Override - public Status delete(String table, String key) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Delete key {} from table {}", key, table); - } - try { - CosmosContainer container = AzureCosmosClient.containerCache.get(table); - if (container == null) { - container = AzureCosmosClient.database.getContainer(table); - AzureCosmosClient.containerCache.put(table, container); - } - container.deleteItem(key, new PartitionKey(key), new CosmosItemRequestOptions()); - - return Status.OK; - } catch (Exception e) { - if (!AzureCosmosClient.includeExceptionStackInLog) { - e = null; - } - LOGGER.error("Failed to delete key {} in collection {}", key, table, e); - } - return Status.ERROR; - } - - private String createSelectTop(Set fields, int top) { - if (fields == null) { - return "SELECT TOP " + top + " * "; - } else { - StringBuilder result = new StringBuilder("SELECT TOP ").append(top).append(" "); - int initLength = result.length(); - for (String field : fields) { - if (result.length() != initLength) { - result.append(", "); - } - result.append("r['").append(field).append("'] "); - } - return result.toString(); - } - } -} diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 314490e888..5f45899659 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -7,9 +7,13 @@ + + + + + - From 487d90e6fca7f2418446a60fb936f3c9359ea8ff Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sun, 31 Jul 2022 00:03:41 -0400 Subject: [PATCH 11/72] Logging marker --- azurecosmos/pom.xml | 7 +++---- .../src/main/java/site/ycsb/db/AzureCosmosClient.java | 11 +++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/azurecosmos/pom.xml b/azurecosmos/pom.xml index 46ce4dd3f6..ff4d0f1026 100644 --- a/azurecosmos/pom.xml +++ b/azurecosmos/pom.xml @@ -30,7 +30,6 @@ LICENSE file. jar - com.azure azure-cosmos @@ -39,17 +38,17 @@ LICENSE file. org.apache.logging.log4j log4j-api - 2.7 + 2.18.0 org.apache.logging.log4j log4j-core - 2.7 + 2.18.0 org.apache.logging.log4j log4j-slf4j-impl - 2.7 + 2.18.0 site.ycsb diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 88e013f6f7..53b6fafec9 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -39,6 +39,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.Marker; +import org.slf4j.MarkerFactory; import site.ycsb.ByteIterator; import site.ycsb.DB; import site.ycsb.DBException; @@ -78,6 +80,7 @@ public class AzureCosmosClient extends DB { private static final String DEFAULT_USER_AGENT = "azurecosmos-ycsb"; private static final Logger LOGGER = LoggerFactory.getLogger(AzureCosmosClient.class); + private static final Marker DIAGNOSTIC = MarkerFactory.getMarker("DIAGNOSTIC"); /** * Count the number of times initialized to teardown on the last @@ -317,7 +320,7 @@ public Status read(String table, String key, Set fields, Map 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(response.getDiagnostics().toString()); + LOGGER.warn(DIAGNOSTIC,response.getDiagnostics().toString()); } return Status.OK; @@ -421,7 +424,7 @@ public Status update(String table, String key, Map values) CosmosItemResponse response = container.patchItem(key, pk, cosmosPatchOperations, ObjectNode.class); if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(response.getDiagnostics().toString()); + LOGGER.warn(DIAGNOSTIC,response.getDiagnostics().toString()); } return Status.OK; @@ -474,7 +477,7 @@ public Status insert(String table, String key, Map values) if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(response.getDiagnostics().toString()); + LOGGER.warn(DIAGNOSTIC,response.getDiagnostics().toString()); } return Status.OK; @@ -505,7 +508,7 @@ public Status delete(String table, String key) { new CosmosItemRequestOptions()); if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(response.getDiagnostics().toString()); + LOGGER.warn(DIAGNOSTIC,response.getDiagnostics().toString()); } return Status.OK; From 449184b6a7153954c953ca3ff9389039d2e49fa6 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sun, 31 Jul 2022 00:21:35 -0400 Subject: [PATCH 12/72] Update AzureCosmosClient.java --- .../src/main/java/site/ycsb/db/AzureCosmosClient.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 53b6fafec9..f8f7cfa260 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -320,7 +320,7 @@ public Status read(String table, String key, Set fields, Map 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(DIAGNOSTIC,response.getDiagnostics().toString()); + LOGGER.warn(DIAGNOSTIC, response.getDiagnostics().toString()); } return Status.OK; @@ -424,7 +424,7 @@ public Status update(String table, String key, Map values) CosmosItemResponse response = container.patchItem(key, pk, cosmosPatchOperations, ObjectNode.class); if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(DIAGNOSTIC,response.getDiagnostics().toString()); + LOGGER.warn(DIAGNOSTIC, response.getDiagnostics().toString()); } return Status.OK; @@ -477,7 +477,7 @@ public Status insert(String table, String key, Map values) if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(DIAGNOSTIC,response.getDiagnostics().toString()); + LOGGER.warn(DIAGNOSTIC, response.getDiagnostics().toString()); } return Status.OK; @@ -508,7 +508,7 @@ public Status delete(String table, String key) { new CosmosItemRequestOptions()); if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(DIAGNOSTIC,response.getDiagnostics().toString()); + LOGGER.warn(DIAGNOSTIC, response.getDiagnostics().toString()); } return Status.OK; From 5434dae306bab36fd6a51c819aa65d16d793a69d Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sun, 31 Jul 2022 07:48:35 -0400 Subject: [PATCH 13/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 5f45899659..66f4e5b14e 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -7,7 +7,7 @@ - + From 53391abd964da0b431b1db994dc0cbe2ca0cdbf9 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sun, 31 Jul 2022 08:53:36 -0400 Subject: [PATCH 14/72] logging --- azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java | 2 +- azurecosmos/src/main/resources/log4j2.xml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index f8f7cfa260..4176b28b17 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -477,7 +477,7 @@ public Status insert(String table, String key, Map values) if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(DIAGNOSTIC, response.getDiagnostics().toString()); + LOGGER.warn(response.getDiagnostics().toString()); } return Status.OK; diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 66f4e5b14e..b5627e3792 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -7,11 +7,9 @@ - - From 7ff417507ae90dd55fa4bceabfad0b1833ee06d2 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sun, 31 Jul 2022 15:06:20 -0400 Subject: [PATCH 15/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index b5627e3792..5a380d3bf0 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -8,7 +8,7 @@ - + @@ -24,6 +24,7 @@ + \ No newline at end of file From 0279b432746d6f98c4e97c0065bd9ac300e605df Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sun, 31 Jul 2022 15:38:13 -0400 Subject: [PATCH 16/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 5a380d3bf0..245b5599c9 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -7,21 +7,17 @@ + + - - - - - - - - - + + + From 0e28ab51295f6a523691cb1733cf5eca22365ee6 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sun, 31 Jul 2022 15:54:01 -0400 Subject: [PATCH 17/72] Update AzureCosmosClient.java --- azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 4176b28b17..f8f7cfa260 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -477,7 +477,7 @@ public Status insert(String table, String key, Map values) if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(response.getDiagnostics().toString()); + LOGGER.warn(DIAGNOSTIC, response.getDiagnostics().toString()); } return Status.OK; From d2a71e22b5b54feeacbc319cf6681597a4cca1b5 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Mon, 1 Aug 2022 21:58:01 -0400 Subject: [PATCH 18/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 245b5599c9..2bb03034cc 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -13,6 +13,20 @@ + + + + + + + + + + @@ -20,7 +34,7 @@ - + \ No newline at end of file From 6623744cc7424d0d8cd02dd0de7a0a3792d04851 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Mon, 1 Aug 2022 22:30:25 -0400 Subject: [PATCH 19/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 2bb03034cc..626ad90a0c 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -9,11 +9,6 @@ - - - - - - From b25f0fdb5b56613d9ed997a4c61e9212748bd2b4 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Mon, 1 Aug 2022 22:52:50 -0400 Subject: [PATCH 20/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 626ad90a0c..0b3377b926 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -11,8 +11,8 @@ From 2512c7ece409d99d01731e05b9d83999624c311b Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Tue, 2 Aug 2022 09:37:45 -0400 Subject: [PATCH 21/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 0b3377b926..0ec368ad33 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -18,7 +18,7 @@ - + From 821f9840f57b8453f1dd48f89b6c63f884aad904 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Tue, 2 Aug 2022 10:05:21 -0400 Subject: [PATCH 22/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 0ec368ad33..2a7d579a14 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -11,14 +11,14 @@ - + From 71ad4ed5c82fa72e8be5b3ed22c022f86c1ba16f Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Wed, 3 Aug 2022 22:05:46 -0400 Subject: [PATCH 23/72] separating diagnostic logs --- .../java/site/ycsb/db/AzureCosmosClient.java | 13 ++++++---- azurecosmos/src/main/resources/log4j2.xml | 25 ++++++++++++++----- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index f8f7cfa260..20066d663a 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -80,7 +80,10 @@ public class AzureCosmosClient extends DB { private static final String DEFAULT_USER_AGENT = "azurecosmos-ycsb"; private static final Logger LOGGER = LoggerFactory.getLogger(AzureCosmosClient.class); - private static final Marker DIAGNOSTIC = MarkerFactory.getMarker("DIAGNOSTIC"); + private static final Marker WRITE_DIAGNOSTIC = MarkerFactory.getMarker("WRITE_DIAGNOSTIC"); + private static final Marker READ_DIAGNOSTIC = MarkerFactory.getMarker("READ_DIAGNOSTIC"); + private static final Marker UPDATE_DIAGNOSTIC = MarkerFactory.getMarker("UPDATE_DIAGNOSTIC"); + private static final Marker DELETE_DIAGNOSTIC = MarkerFactory.getMarker("DELETE_DIAGNOSTIC"); /** * Count the number of times initialized to teardown on the last @@ -320,7 +323,7 @@ public Status read(String table, String key, Set fields, Map 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(DIAGNOSTIC, response.getDiagnostics().toString()); + LOGGER.warn(READ_DIAGNOSTIC, response.getDiagnostics().toString()); } return Status.OK; @@ -424,7 +427,7 @@ public Status update(String table, String key, Map values) CosmosItemResponse response = container.patchItem(key, pk, cosmosPatchOperations, ObjectNode.class); if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(DIAGNOSTIC, response.getDiagnostics().toString()); + LOGGER.warn(UPDATE_DIAGNOSTIC, response.getDiagnostics().toString()); } return Status.OK; @@ -477,7 +480,7 @@ public Status insert(String table, String key, Map values) if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(DIAGNOSTIC, response.getDiagnostics().toString()); + LOGGER.warn(WRITE_DIAGNOSTIC, response.getDiagnostics().toString()); } return Status.OK; @@ -508,7 +511,7 @@ public Status delete(String table, String key) { new CosmosItemRequestOptions()); if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(DIAGNOSTIC, response.getDiagnostics().toString()); + LOGGER.warn(DELETE_DIAGNOSTIC, response.getDiagnostics().toString()); } return Status.OK; diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 2a7d579a14..cbe5499911 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -10,15 +10,28 @@ - + - + + + + + + + + + + @@ -28,7 +41,7 @@ - + \ No newline at end of file From 7d95703a8015deaeaafafcd6b5372b4b88e88272 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh Date: Thu, 4 Aug 2022 16:52:45 -0400 Subject: [PATCH 24/72] Adding long number support for long run --- core/src/main/java/site/ycsb/Client.java | 12 ++++++------ core/src/main/java/site/ycsb/ClientThread.java | 10 +++++----- .../main/java/site/ycsb/workloads/CoreWorkload.java | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/site/ycsb/Client.java b/core/src/main/java/site/ycsb/Client.java index bd96821170..990ff9b1e8 100644 --- a/core/src/main/java/site/ycsb/Client.java +++ b/core/src/main/java/site/ycsb/Client.java @@ -408,18 +408,18 @@ private static List initDb(String dbname, Properties props, int th final List clients = new ArrayList<>(threadcount); try (final TraceScope span = tracer.newScope(CLIENT_INIT_SPAN)) { - int opcount; + long opcount; if (dotransactions) { - opcount = Integer.parseInt(props.getProperty(OPERATION_COUNT_PROPERTY, "0")); + opcount = Long.parseLong(props.getProperty(OPERATION_COUNT_PROPERTY, "0")); } else { if (props.containsKey(INSERT_COUNT_PROPERTY)) { - opcount = Integer.parseInt(props.getProperty(INSERT_COUNT_PROPERTY, "0")); + opcount = Long.parseLong(props.getProperty(INSERT_COUNT_PROPERTY, "0")); } else { - opcount = Integer.parseInt(props.getProperty(RECORD_COUNT_PROPERTY, DEFAULT_RECORD_COUNT)); + opcount = Long.parseLong(props.getProperty(RECORD_COUNT_PROPERTY, DEFAULT_RECORD_COUNT)); } } if (threadcount > opcount && opcount > 0){ - threadcount = opcount; + threadcount = (int) opcount; System.out.println("Warning: the threadcount is bigger than recordcount, the threadcount will be recordcount!"); } for (int threadid = 0; threadid < threadcount; threadid++) { @@ -432,7 +432,7 @@ private static List initDb(String dbname, Properties props, int th break; } - int threadopcount = opcount / threadcount; + long threadopcount = opcount / threadcount; // ensure correct number of operations, in case opcount is not a multiple of threadcount if (threadid < opcount % threadcount) { diff --git a/core/src/main/java/site/ycsb/ClientThread.java b/core/src/main/java/site/ycsb/ClientThread.java index fcfc13267d..20dacf2fd7 100644 --- a/core/src/main/java/site/ycsb/ClientThread.java +++ b/core/src/main/java/site/ycsb/ClientThread.java @@ -34,7 +34,7 @@ public class ClientThread implements Runnable { private DB db; private boolean dotransactions; private Workload workload; - private int opcount; + private long opcount; private double targetOpsPerMs; private int opsdone; @@ -56,7 +56,7 @@ public class ClientThread implements Runnable { * @param targetperthreadperms target number of operations per thread per ms * @param completeLatch The latch tracking the completion of all clients. */ - public ClientThread(DB db, boolean dotransactions, Workload workload, Properties props, int opcount, + public ClientThread(DB db, boolean dotransactions, Workload workload, Properties props, long opcount, double targetperthreadperms, CountDownLatch completeLatch) { this.db = db; this.dotransactions = dotransactions; @@ -81,7 +81,7 @@ public void setThreadCount(final int threadCount) { threadcount = threadCount; } - public int getOpsDone() { + public long getOpsDone() { return opsdone; } @@ -179,8 +179,8 @@ private void throttleNanos(long startTimeNanos) { /** * The total amount of work this thread is still expected to do. */ - int getOpsTodo() { - int todo = opcount - opsdone; + long getOpsTodo() { + long todo = opcount - opsdone; return todo < 0 ? 0 : todo; } } diff --git a/core/src/main/java/site/ycsb/workloads/CoreWorkload.java b/core/src/main/java/site/ycsb/workloads/CoreWorkload.java index 082cb0dc2b..29a9889f3a 100644 --- a/core/src/main/java/site/ycsb/workloads/CoreWorkload.java +++ b/core/src/main/java/site/ycsb/workloads/CoreWorkload.java @@ -448,7 +448,7 @@ public void init(Properties p) throws WorkloadException { long insertstart = Long.parseLong(p.getProperty(INSERT_START_PROPERTY, INSERT_START_PROPERTY_DEFAULT)); long insertcount= - Integer.parseInt(p.getProperty(INSERT_COUNT_PROPERTY, String.valueOf(recordcount - insertstart))); + Long.parseLong(p.getProperty(INSERT_COUNT_PROPERTY, String.valueOf(recordcount - insertstart))); // Confirm valid values for insertstart and insertcount in relation to recordcount if (recordcount < (insertstart + insertcount)) { System.err.println("Invalid combination of insertstart, insertcount and recordcount."); From 60b6826b5d0f37e1e74b7ab4fabda4837571b0ce Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 5 Aug 2022 10:58:24 -0400 Subject: [PATCH 25/72] adding multiple markers --- azurecosmos/src/main/resources/log4j2.xml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index cbe5499911..9bb6553fe8 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -7,12 +7,13 @@ - + + @@ -24,8 +25,8 @@ From d4e206aae4878b6dedf192a8bc778638a3020154 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 5 Aug 2022 11:08:54 -0400 Subject: [PATCH 26/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 9bb6553fe8..e960b2be47 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -8,7 +8,7 @@ - + + \ No newline at end of file From 99ffe0c5c9b7dfd1da863458576a72b81ce11937 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 5 Aug 2022 12:02:26 -0400 Subject: [PATCH 27/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index e960b2be47..26f7221541 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -12,8 +12,8 @@ @@ -25,8 +25,8 @@ From f5d27241f90916fe4c52344c658b1851e1d9e45d Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 5 Aug 2022 14:34:16 -0400 Subject: [PATCH 28/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 26f7221541..f22937a7f0 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -7,8 +7,10 @@ - + + + Date: Fri, 5 Aug 2022 16:30:17 -0400 Subject: [PATCH 29/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index f22937a7f0..839feca8d7 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -8,8 +8,8 @@ + - Date: Fri, 5 Aug 2022 16:39:49 -0400 Subject: [PATCH 30/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 839feca8d7..4bbe8a5ab0 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -8,7 +8,7 @@ - + From 24a053ec345dff07ab9874be3a3d89a13e108446 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 5 Aug 2022 17:31:00 -0400 Subject: [PATCH 31/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 4bbe8a5ab0..1bdb580e0e 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -9,7 +9,7 @@ - + - + @@ -34,7 +34,7 @@ - + From 2cde15facfadf4129374a4f826b1fbee81c02e11 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 5 Aug 2022 17:38:37 -0400 Subject: [PATCH 32/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 1bdb580e0e..a05732bc43 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -7,10 +7,10 @@ - + Date: Fri, 5 Aug 2022 18:36:18 -0400 Subject: [PATCH 33/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index a05732bc43..1bdb580e0e 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -7,10 +7,10 @@ - + Date: Fri, 5 Aug 2022 20:57:22 -0400 Subject: [PATCH 34/72] changing marker name --- .../main/java/site/ycsb/db/AzureCosmosClient.java | 4 ++-- azurecosmos/src/main/resources/log4j2.xml | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 20066d663a..48a0c93b00 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -80,7 +80,7 @@ public class AzureCosmosClient extends DB { private static final String DEFAULT_USER_AGENT = "azurecosmos-ycsb"; private static final Logger LOGGER = LoggerFactory.getLogger(AzureCosmosClient.class); - private static final Marker WRITE_DIAGNOSTIC = MarkerFactory.getMarker("WRITE_DIAGNOSTIC"); + private static final Marker CREATE_DIAGNOSTIC = MarkerFactory.getMarker("CREATE_DIAGNOSTIC"); private static final Marker READ_DIAGNOSTIC = MarkerFactory.getMarker("READ_DIAGNOSTIC"); private static final Marker UPDATE_DIAGNOSTIC = MarkerFactory.getMarker("UPDATE_DIAGNOSTIC"); private static final Marker DELETE_DIAGNOSTIC = MarkerFactory.getMarker("DELETE_DIAGNOSTIC"); @@ -480,7 +480,7 @@ public Status insert(String table, String key, Map values) if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(WRITE_DIAGNOSTIC, response.getDiagnostics().toString()); + LOGGER.warn(CREATE_DIAGNOSTIC, response.getDiagnostics().toString()); } return Status.OK; diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 1bdb580e0e..3f2a6f41c4 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -9,32 +9,32 @@ - + - + - + - + From d07f11c10ce1a88eb399ea2564fbbed34c7bb09e Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 5 Aug 2022 21:22:13 -0400 Subject: [PATCH 35/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 3f2a6f41c4..7260fd2849 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -14,8 +14,8 @@ @@ -27,8 +27,8 @@ From ab8a5ae03f52490417f4513db90d502b7be488a9 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 5 Aug 2022 21:41:51 -0400 Subject: [PATCH 36/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 7260fd2849..f9a8a1fd93 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -21,20 +21,20 @@ - + - + From 3c01e91addd16e3eecb534b2991d2685941b8591 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 5 Aug 2022 21:43:52 -0400 Subject: [PATCH 37/72] Update log4j2.xml --- azurecosmos/src/main/resources/log4j2.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index f9a8a1fd93..208bb4bab0 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -21,7 +21,7 @@ - + @@ -34,7 +34,7 @@ - + From 45e2b26561ab90595191d50371b34f282705b490 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 5 Aug 2022 22:18:04 -0400 Subject: [PATCH 38/72] query and patch markets added --- .../java/site/ycsb/db/AzureCosmosClient.java | 5 ++-- azurecosmos/src/main/resources/log4j2.xml | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 48a0c93b00..aff67aaa0a 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -82,8 +82,9 @@ public class AzureCosmosClient extends DB { private static final Logger LOGGER = LoggerFactory.getLogger(AzureCosmosClient.class); private static final Marker CREATE_DIAGNOSTIC = MarkerFactory.getMarker("CREATE_DIAGNOSTIC"); private static final Marker READ_DIAGNOSTIC = MarkerFactory.getMarker("READ_DIAGNOSTIC"); - private static final Marker UPDATE_DIAGNOSTIC = MarkerFactory.getMarker("UPDATE_DIAGNOSTIC"); + private static final Marker PATCH_DIAGNOSTIC = MarkerFactory.getMarker("PATCH_DIAGNOSTIC"); private static final Marker DELETE_DIAGNOSTIC = MarkerFactory.getMarker("DELETE_DIAGNOSTIC"); + private static final Marker QUERY_DIAGNOSTIC = MarkerFactory.getMarker("DELETE_DIAGNOSTIC"); /** * Count the number of times initialized to teardown on the last @@ -427,7 +428,7 @@ public Status update(String table, String key, Map values) CosmosItemResponse response = container.patchItem(key, pk, cosmosPatchOperations, ObjectNode.class); if (diagnosticsLatencyThresholdInMS > 0 && response.getDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(UPDATE_DIAGNOSTIC, response.getDiagnostics().toString()); + LOGGER.warn(PATCH_DIAGNOSTIC, response.getDiagnostics().toString()); } return Status.OK; diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 208bb4bab0..f1401e8383 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -9,6 +9,8 @@ + + @@ -37,6 +39,32 @@ + + + + + + + + + + + + + + + + + + @@ -46,6 +74,8 @@ + + \ No newline at end of file From d1200d572340b78afa7054dce0bdd880e77b30af Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sat, 6 Aug 2022 11:43:48 -0400 Subject: [PATCH 39/72] Enabling request diagnostics for query --- .../src/main/java/site/ycsb/db/AzureCosmosClient.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index aff67aaa0a..235aab287c 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -374,8 +374,14 @@ public Status scan(String table, String startkey, int recordcount, Set f CosmosPagedIterable pagedIterable = container.queryItems(querySpec, queryOptions, ObjectNode.class); Iterator> pageIterator = pagedIterable .iterableByPage(AzureCosmosClient.preferredPageSize).iterator(); - while (pageIterator.hasNext()) { - List pageDocs = pageIterator.next().getResults(); + while (pageIterator.hasNext()) { + FeedResponse feedResponse = pageIterator.next(); + feedResponse.getCosmosDiagnostics().getDuration(); + if (diagnosticsLatencyThresholdInMS > 0 && + feedResponse.getCosmosDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { + LOGGER.warn(QUERY_DIAGNOSTIC, feedResponse.getCosmosDiagnostics().getDuration().toString()); + } + List pageDocs = feedResponse.getResults(); for (ObjectNode doc : pageDocs) { Map stringResults = new HashMap<>(doc.size()); Iterator> nodeIterator = doc.fields(); From 0f1658893196519639edecec5b2854e38b515d9a Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sat, 6 Aug 2022 12:08:33 -0400 Subject: [PATCH 40/72] UUID as ID for insert --- azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 235aab287c..4bdc146874 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -471,10 +471,12 @@ public Status insert(String table, String key, Map values) container = AzureCosmosClient.database.getContainer(table); AzureCosmosClient.containerCache.put(table, container); } + key = String.valueOf(java.util.UUID.randomUUID()); PartitionKey pk = new PartitionKey(key); ObjectNode node = OBJECT_MAPPER.createObjectNode(); node.put("id", key); + for (Map.Entry pair : values.entrySet()) { node.put(pair.getKey(), pair.getValue().toString()); } From cb4ff2024851c6722c68e77f00e192907e4cd302 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sat, 6 Aug 2022 15:45:03 -0400 Subject: [PATCH 41/72] Update AzureCosmosClient.java --- .../java/site/ycsb/db/AzureCosmosClient.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 4bdc146874..9717bbaa99 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -343,7 +343,6 @@ public Status read(String table, String key, Set fields, Map f CosmosPagedIterable pagedIterable = container.queryItems(querySpec, queryOptions, ObjectNode.class); Iterator> pageIterator = pagedIterable .iterableByPage(AzureCosmosClient.preferredPageSize).iterator(); - while (pageIterator.hasNext()) { - FeedResponse feedResponse = pageIterator.next(); - feedResponse.getCosmosDiagnostics().getDuration(); - if (diagnosticsLatencyThresholdInMS > 0 && - feedResponse.getCosmosDiagnostics().getDuration().compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(QUERY_DIAGNOSTIC, feedResponse.getCosmosDiagnostics().getDuration().toString()); - } - List pageDocs = feedResponse.getResults(); + while (pageIterator.hasNext()) { + FeedResponse feedResponse = pageIterator.next(); + feedResponse.getCosmosDiagnostics().getDuration(); + if (diagnosticsLatencyThresholdInMS > 0 && + feedResponse.getCosmosDiagnostics().getDuration() + .compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { + LOGGER.warn(QUERY_DIAGNOSTIC, feedResponse.getCosmosDiagnostics().getDuration().toString()); + } + List pageDocs = feedResponse.getResults(); for (ObjectNode doc : pageDocs) { Map stringResults = new HashMap<>(doc.size()); Iterator> nodeIterator = doc.fields(); From 721e5b68fd4144dc34eed05556a822111d3e8ce9 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sat, 6 Aug 2022 21:22:05 -0400 Subject: [PATCH 42/72] Update AzureCosmosClient.java --- azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 9717bbaa99..c6fc3051d8 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -471,7 +471,7 @@ public Status insert(String table, String key, Map values) container = AzureCosmosClient.database.getContainer(table); AzureCosmosClient.containerCache.put(table, container); } - key = String.valueOf(java.util.UUID.randomUUID()); + // key = String.valueOf(java.util.UUID.randomUUID()); PartitionKey pk = new PartitionKey(key); ObjectNode node = OBJECT_MAPPER.createObjectNode(); From 227bea0010ab0591c12cc926881550d3fc980e1c Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Sat, 6 Aug 2022 22:22:25 -0400 Subject: [PATCH 43/72] Update AzureCosmosClient.java --- azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index c6fc3051d8..9717bbaa99 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -471,7 +471,7 @@ public Status insert(String table, String key, Map values) container = AzureCosmosClient.database.getContainer(table); AzureCosmosClient.containerCache.put(table, container); } - // key = String.valueOf(java.util.UUID.randomUUID()); + key = String.valueOf(java.util.UUID.randomUUID()); PartitionKey pk = new PartitionKey(key); ObjectNode node = OBJECT_MAPPER.createObjectNode(); From dee71229354dfa395c0256c7f1da5614c0f72d36 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Mon, 8 Aug 2022 18:32:57 -0400 Subject: [PATCH 44/72] Upgrade to Log4J2 and logging for request diagnostics --- .../java/site/ycsb/db/AzureCosmosClient.java | 9 +-------- azurecosmos/src/main/resources/log4j2.xml | 18 +++++++++--------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 9717bbaa99..8a8d834fd2 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -84,7 +84,7 @@ public class AzureCosmosClient extends DB { private static final Marker READ_DIAGNOSTIC = MarkerFactory.getMarker("READ_DIAGNOSTIC"); private static final Marker PATCH_DIAGNOSTIC = MarkerFactory.getMarker("PATCH_DIAGNOSTIC"); private static final Marker DELETE_DIAGNOSTIC = MarkerFactory.getMarker("DELETE_DIAGNOSTIC"); - private static final Marker QUERY_DIAGNOSTIC = MarkerFactory.getMarker("DELETE_DIAGNOSTIC"); + private static final Marker QUERY_DIAGNOSTIC = MarkerFactory.getMarker("QUERY_DIAGNOSTIC"); /** * Count the number of times initialized to teardown on the last @@ -375,12 +375,6 @@ public Status scan(String table, String startkey, int recordcount, Set f .iterableByPage(AzureCosmosClient.preferredPageSize).iterator(); while (pageIterator.hasNext()) { FeedResponse feedResponse = pageIterator.next(); - feedResponse.getCosmosDiagnostics().getDuration(); - if (diagnosticsLatencyThresholdInMS > 0 && - feedResponse.getCosmosDiagnostics().getDuration() - .compareTo(Duration.ofMillis(diagnosticsLatencyThresholdInMS)) > 0) { - LOGGER.warn(QUERY_DIAGNOSTIC, feedResponse.getCosmosDiagnostics().getDuration().toString()); - } List pageDocs = feedResponse.getResults(); for (ObjectNode doc : pageDocs) { Map stringResults = new HashMap<>(doc.size()); @@ -471,7 +465,6 @@ public Status insert(String table, String key, Map values) container = AzureCosmosClient.database.getContainer(table); AzureCosmosClient.containerCache.put(table, container); } - key = String.valueOf(java.util.UUID.randomUUID()); PartitionKey pk = new PartitionKey(key); ObjectNode node = OBJECT_MAPPER.createObjectNode(); diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index f1401e8383..751963384a 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -8,22 +8,22 @@ - - - - + + + + - + @@ -36,7 +36,7 @@ - + @@ -49,7 +49,7 @@ - + @@ -62,7 +62,7 @@ - + From 3fcf68c676989c6b6e0130b9e40c08cf97d779af Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh Date: Tue, 9 Aug 2022 16:14:24 -0400 Subject: [PATCH 45/72] Updating cosmos sdk to latest 4.34.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 28e8760b80..245fef0e04 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ LICENSE file. 3.1.2 4.4.1 1.8.2 - 4.28.0 + 4.34.0 4.0.0 3.0.0 2.0.1 From 2398fd82c06becea41e585b1ef8772e6d223460e Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Tue, 9 Aug 2022 18:41:03 -0400 Subject: [PATCH 46/72] Revert "Merge pull request #7 from Azure/users/nakumars/updateCosmosSDKTo4.34.0" This reverts commit 747f84d462100b57619b838a330acf1c2e95c2f9, reversing changes made to 89e9e8f35b891582c9bdeb5be91df86d10fe70cb. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 245fef0e04..28e8760b80 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ LICENSE file. 3.1.2 4.4.1 1.8.2 - 4.34.0 + 4.28.0 4.0.0 3.0.0 2.0.1 From f56210ae5b922db4ab3fb858331dc2816eb80ff0 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Tue, 9 Aug 2022 18:42:54 -0400 Subject: [PATCH 47/72] Cosmos sdk version update to 4.34.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 28e8760b80..245fef0e04 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ LICENSE file. 3.1.2 4.4.1 1.8.2 - 4.28.0 + 4.34.0 4.0.0 3.0.0 2.0.1 From 743806b85d0ff075f69982c91740b902c9e67beb Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh Date: Thu, 25 Aug 2022 15:06:31 -0400 Subject: [PATCH 48/72] Adding AppInsight monitering support --- azurecosmos/conf/azurecosmos.properties | 5 +- azurecosmos/pom.xml | 5 + .../java/site/ycsb/db/AzureCosmosClient.java | 154 +++++++++++++++++- 3 files changed, 162 insertions(+), 2 deletions(-) diff --git a/azurecosmos/conf/azurecosmos.properties b/azurecosmos/conf/azurecosmos.properties index 389ba7ba93..2b4b42e18d 100644 --- a/azurecosmos/conf/azurecosmos.properties +++ b/azurecosmos/conf/azurecosmos.properties @@ -88,4 +88,7 @@ # azurecosmos.preferredPageSize = -1 # output file location -# exportfile = \ No newline at end of file +# exportfile = + +# application insight connection String +# azurecosmos.appInsightConnectionString = diff --git a/azurecosmos/pom.xml b/azurecosmos/pom.xml index ff4d0f1026..5da05181b3 100644 --- a/azurecosmos/pom.xml +++ b/azurecosmos/pom.xml @@ -56,5 +56,10 @@ LICENSE file. ${project.version} provided + + io.micrometer + micrometer-registry-azure-monitor + 1.9.2 + diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 8a8d834fd2..b2fdf47fdc 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -37,6 +37,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.microsoft.applicationinsights.TelemetryConfiguration; +import io.micrometer.azuremonitor.AzureMonitorConfig; +import io.micrometer.azuremonitor.AzureMonitorMeterRegistry; +import io.micrometer.core.instrument.Clock; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Timer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.Marker; @@ -57,6 +63,7 @@ import java.util.Set; import java.util.Vector; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; /** @@ -103,6 +110,24 @@ public class AzureCosmosClient extends DB { private static boolean includeExceptionStackInLog; private static Map containerCache; private static String userAgent; + + private AzureMonitorMeterRegistry azureMonitorMeterRegistry; + + private volatile Counter readSuccessCounter; + private volatile Counter readFailureCounter; + private volatile Timer readSuccessLatencyTimer; + + private volatile Counter scanSuccessCounter; + private volatile Counter scanFailureCounter; + private volatile Timer scanSuccessLatencyTimer; + + private volatile Counter writeSuccessCounter; + private volatile Counter writeFailureCounter; + private volatile Timer writeSuccessLatencyTimer; + + private volatile Counter updateSuccessCounter; + private volatile Counter updateFailureCounter; + private volatile Timer updateSuccessLatencyTimer; @Override public void init() throws DBException { @@ -234,6 +259,12 @@ private void initAzureCosmosClient() throws DBException { throw new DBException( "Invalid database name (" + AzureCosmosClient.databaseName + ") or failed to read database.", e); } + + String appInsightConnectionString = this.getStringProperty("azurecosmos.appInsightConnectionString", null); + if (appInsightConnectionString != null) { + this.azureMonitorMeterRegistry = this.azureMonitorMeterRegistry(appInsightConnectionString); + registerMeter(); + } } private String getStringProperty(String propertyName, String defaultValue) { @@ -260,6 +291,18 @@ private int getIntProperty(String propertyName, int defaultValue) { } } + private double getDoubleProperty(String propertyName, double defaultValue) { + String stringVal = getProperties().getProperty(propertyName, null); + if (stringVal == null) { + return defaultValue; + } + try { + return Double.parseDouble(stringVal); + } catch (NumberFormatException e) { + return defaultValue; + } + } + /** * Cleanup any state for this DB. Called once per DB instance; there is one DB * instance per client thread. @@ -295,6 +338,7 @@ public void cleanup() throws DBException { @Override public Status read(String table, String key, Set fields, Map result) { try { + long st = System.nanoTime(); CosmosContainer container = AzureCosmosClient.containerCache.get(table); if (container == null) { container = AzureCosmosClient.database.getContainer(table); @@ -327,6 +371,12 @@ public Status read(String table, String key, Set fields, Map fields, Map fields, Map fields, Vector> result) { try { + long st = System.nanoTime(); CosmosQueryRequestOptions queryOptions = new CosmosQueryRequestOptions(); queryOptions.setMaxDegreeOfParallelism(AzureCosmosClient.maxDegreeOfParallelism); queryOptions.setMaxBufferedItemCount(AzureCosmosClient.maxBufferedItemCount); @@ -388,6 +442,13 @@ public Status scan(String table, String startkey, int recordcount, Set f result.add(byteResults); } } + + if (scanSuccessLatencyTimer != null) { + long en = System.nanoTime(); + long latency = (en - st) / 1000; + scanSuccessLatencyTimer.record(latency, TimeUnit.MICROSECONDS); + scanSuccessCounter.increment(); + } return Status.OK; } catch (CosmosException e) { int statusCode = e.getStatusCode(); @@ -397,6 +458,9 @@ public Status scan(String table, String startkey, int recordcount, Set f LOGGER.error("Failed to query key {} from collection {} in database {} statusCode {}", startkey, table, AzureCosmosClient.databaseName, statusCode, e); } + if (scanFailureCounter != null) { + scanFailureCounter.increment(); + } return Status.ERROR; } @@ -413,6 +477,7 @@ public Status scan(String table, String startkey, int recordcount, Set f @Override public Status update(String table, String key, Map values) { try { + long st = System.nanoTime(); CosmosContainer container = AzureCosmosClient.containerCache.get(table); if (container == null) { container = AzureCosmosClient.database.getContainer(table); @@ -431,6 +496,12 @@ public Status update(String table, String key, Map values) LOGGER.warn(PATCH_DIAGNOSTIC, response.getDiagnostics().toString()); } + if (updateSuccessLatencyTimer != null) { + long en = System.nanoTime(); + long latency = (en - st) / 1000; + updateSuccessLatencyTimer.record(latency, TimeUnit.MICROSECONDS); + updateSuccessCounter.increment(); + } return Status.OK; } catch (CosmosException e) { int statusCode = e.getStatusCode(); @@ -441,6 +512,9 @@ public Status update(String table, String key, Map values) AzureCosmosClient.databaseName, statusCode, e); } + if (updateFailureCounter != null) { + updateFailureCounter.increment(); + } return Status.ERROR; } @@ -458,7 +532,7 @@ public Status insert(String table, String key, Map values) if (LOGGER.isDebugEnabled()) { LOGGER.debug("Insert key: {} into table: {}", key, table); } - + long st = System.nanoTime(); try { CosmosContainer container = AzureCosmosClient.containerCache.get(table); if (container == null) { @@ -485,6 +559,12 @@ public Status insert(String table, String key, Map values) LOGGER.warn(CREATE_DIAGNOSTIC, response.getDiagnostics().toString()); } + if (writeSuccessLatencyTimer != null) { + long en = System.nanoTime(); + long latency = (en - st) / 1000; + writeSuccessLatencyTimer.record(latency, TimeUnit.MICROSECONDS); + writeSuccessCounter.increment(); + } return Status.OK; } catch (CosmosException e) { int statusCode = e.getStatusCode(); @@ -494,6 +574,9 @@ public Status insert(String table, String key, Map values) LOGGER.error("Failed to insert key {} to collection {} in database {} statusCode {}", key, table, AzureCosmosClient.databaseName, statusCode, e); } + if (writeFailureCounter != null) { + writeFailureCounter.increment(); + } return Status.ERROR; } @@ -543,4 +626,73 @@ private String createSelectTop(Set fields, int top) { return result.toString(); } } + + private synchronized AzureMonitorMeterRegistry azureMonitorMeterRegistry(String appInsightConnectionString) { + if (this.azureMonitorMeterRegistry == null) { + Duration step = Duration.ofSeconds(Integer.getInteger("azure.cosmos.monitoring.azureMonitor.step", 10)); + boolean enabled = !Boolean.getBoolean("azure.cosmos.monitoring.azureMonitor.disabled"); + final AzureMonitorConfig config = new AzureMonitorConfig() { + @Override + public String get(String key) { + return null; + } + + @Override + public Duration step() { + return step; + } + + @Override + public boolean enabled() { + return enabled; + } + }; + TelemetryConfiguration telemetryConfiguration = TelemetryConfiguration.createDefault(); + telemetryConfiguration.setConnectionString(appInsightConnectionString); + azureMonitorMeterRegistry = AzureMonitorMeterRegistry + .builder(config) + .clock(Clock.SYSTEM) + .telemetryConfiguration(telemetryConfiguration) + .build(); + } + return this.azureMonitorMeterRegistry; + } + + private void registerMeter() { + if (this.getDoubleProperty("readproportion", 0) > 0) { + readSuccessCounter = this.azureMonitorMeterRegistry.counter("Read Successful Operations"); + readFailureCounter = this.azureMonitorMeterRegistry.counter("Read Unsuccessful Operations"); + readSuccessLatencyTimer = Timer.builder("Read Successful Latency") + .publishPercentiles(0.5, 0.95, 0.99, 0.999, 0.9999) + .publishPercentileHistogram() + .register(this.azureMonitorMeterRegistry); + } + + if (this.getDoubleProperty("insertproportion", 0) > 0) { + writeSuccessCounter = this.azureMonitorMeterRegistry.counter("Write Successful Operations"); + writeFailureCounter = this.azureMonitorMeterRegistry.counter("Write Unsuccessful Operations"); + writeSuccessLatencyTimer = Timer.builder("Write Successful Latency") + .publishPercentiles(0.5, 0.95, 0.99, 0.999, 0.9999) + .publishPercentileHistogram() + .register(this.azureMonitorMeterRegistry); + } + + if (this.getDoubleProperty("scanproportion", 0) > 0) { + scanSuccessCounter = this.azureMonitorMeterRegistry.counter("Scan Successful Operations"); + scanFailureCounter = this.azureMonitorMeterRegistry.counter("Scan Unsuccessful Operations"); + scanSuccessLatencyTimer = Timer.builder("Scan Successful Latency") + .publishPercentiles(0.5, 0.95, 0.99, 0.999, 0.9999) + .publishPercentileHistogram() + .register(this.azureMonitorMeterRegistry); + } + + if (this.getDoubleProperty("updateproportion", 0) > 0) { + updateSuccessCounter = this.azureMonitorMeterRegistry.counter("Update Successful Operations"); + updateFailureCounter = this.azureMonitorMeterRegistry.counter("Update Unsuccessful Operations"); + updateSuccessLatencyTimer = Timer.builder("Update Successful Latency") + .publishPercentiles(0.5, 0.95, 0.99, 0.999, 0.9999) + .publishPercentileHistogram() + .register(this.azureMonitorMeterRegistry); + } + } } \ No newline at end of file From 2582c93d8af28c6e78fa7eee4b253fbd4a2dc73d Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 16 Sep 2022 11:09:37 -0400 Subject: [PATCH 49/72] directing Cosmos operation exception to separate file --- .../java/site/ycsb/db/AzureCosmosClient.java | 18 ++- azurecosmos/src/main/resources/log4j2.xml | 118 ++++++++++++++++-- 2 files changed, 117 insertions(+), 19 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index b2fdf47fdc..40f0c56e69 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -92,6 +92,12 @@ public class AzureCosmosClient extends DB { private static final Marker PATCH_DIAGNOSTIC = MarkerFactory.getMarker("PATCH_DIAGNOSTIC"); private static final Marker DELETE_DIAGNOSTIC = MarkerFactory.getMarker("DELETE_DIAGNOSTIC"); private static final Marker QUERY_DIAGNOSTIC = MarkerFactory.getMarker("QUERY_DIAGNOSTIC"); + private static final Marker CREATE_EXCEPTION = MarkerFactory.getMarker("CREATE_EXCEPTION"); + private static final Marker READ_EXCEPTION = MarkerFactory.getMarker("READ_EXCEPTION"); + private static final Marker PATCH_EXCEPTION = MarkerFactory.getMarker("PATCH_EXCEPTION"); + private static final Marker DELETE_EXCEPTION = MarkerFactory.getMarker("DELETE_DIAGNOSTIC"); + private static final Marker QUERY_EXCEPTION = MarkerFactory.getMarker("QUERY_EXCEPTION"); + /** * Count the number of times initialized to teardown on the last @@ -383,7 +389,7 @@ public Status read(String table, String key, Set fields, Map f if (!AzureCosmosClient.includeExceptionStackInLog) { e = null; } - LOGGER.error("Failed to query key {} from collection {} in database {} statusCode {}", startkey, table, + LOGGER.error(QUERY_EXCEPTION, "Failed to query key {} from collection {} in database {} statusCode {}", startkey, table, AzureCosmosClient.databaseName, statusCode, e); } if (scanFailureCounter != null) { @@ -508,7 +514,7 @@ public Status update(String table, String key, Map values) if (!AzureCosmosClient.includeExceptionStackInLog) { e = null; } - LOGGER.error("Failed to update key {} to collection {} in database {} statusCode {}", key, table, + LOGGER.error(PATCH_EXCEPTION, "Failed to update key {} to collection {} in database {} statusCode {}", key, table, AzureCosmosClient.databaseName, statusCode, e); } @@ -571,8 +577,8 @@ public Status insert(String table, String key, Map values) if (!AzureCosmosClient.includeExceptionStackInLog) { e = null; } - LOGGER.error("Failed to insert key {} to collection {} in database {} statusCode {}", key, table, - AzureCosmosClient.databaseName, statusCode, e); + LOGGER.error(CREATE_EXCEPTION, "Failed to insert key {} to collection {} in database {} statusCode {}", key, + table, AzureCosmosClient.databaseName, statusCode, e); } if (writeFailureCounter != null) { writeFailureCounter.increment(); @@ -605,7 +611,7 @@ public Status delete(String table, String key) { if (!AzureCosmosClient.includeExceptionStackInLog) { e = null; } - LOGGER.error("Failed to delete key {} in collection {} database {} statusCode {}", key, table, + LOGGER.error(DELETE_EXCEPTION, "Failed to delete key {} in collection {} database {} statusCode {}", key, table, AzureCosmosClient.databaseName, statusCode, e); } return Status.ERROR; diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index 751963384a..f38cf69409 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -2,22 +2,29 @@ %-4r [%t] %-5p %c %x -%m%n + /tmp/cosmos_client_logs/diagnostics + /tmp/cosmos_client_logs/exceptions - - - + + + + + + + + @@ -29,8 +36,8 @@ @@ -42,8 +49,8 @@ @@ -55,8 +62,8 @@ @@ -65,6 +72,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -72,10 +158,16 @@ - + + + + + + + \ No newline at end of file From 436e65da6f71a7948a9a357efd66870ee4642447 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Fri, 16 Sep 2022 14:24:41 -0400 Subject: [PATCH 50/72] check style fix --- azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 40f0c56e69..0e3deb1efd 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -461,8 +461,8 @@ public Status scan(String table, String startkey, int recordcount, Set f if (!AzureCosmosClient.includeExceptionStackInLog) { e = null; } - LOGGER.error(QUERY_EXCEPTION, "Failed to query key {} from collection {} in database {} statusCode {}", startkey, table, - AzureCosmosClient.databaseName, statusCode, e); + LOGGER.error(QUERY_EXCEPTION, "Failed to query key {} from collection {} in database {} statusCode {}", + startkey, table, AzureCosmosClient.databaseName, statusCode, e); } if (scanFailureCounter != null) { scanFailureCounter.increment(); From 50539734cd01d84247f28c03bd881b263db071f4 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh Date: Thu, 20 Oct 2022 12:34:36 -0400 Subject: [PATCH 51/72] Fixing threading issue on micrometer integration on cosmos ycsm --- .../java/site/ycsb/db/AzureCosmosClient.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 0e3deb1efd..cc161f1cec 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -117,23 +117,23 @@ public class AzureCosmosClient extends DB { private static Map containerCache; private static String userAgent; - private AzureMonitorMeterRegistry azureMonitorMeterRegistry; + private static AzureMonitorMeterRegistry azureMonitorMeterRegistry; - private volatile Counter readSuccessCounter; - private volatile Counter readFailureCounter; - private volatile Timer readSuccessLatencyTimer; + private static Counter readSuccessCounter; + private static Counter readFailureCounter; + private static Timer readSuccessLatencyTimer; - private volatile Counter scanSuccessCounter; - private volatile Counter scanFailureCounter; - private volatile Timer scanSuccessLatencyTimer; + private static Counter scanSuccessCounter; + private static Counter scanFailureCounter; + private static Timer scanSuccessLatencyTimer; - private volatile Counter writeSuccessCounter; - private volatile Counter writeFailureCounter; - private volatile Timer writeSuccessLatencyTimer; + private static Counter writeSuccessCounter; + private static Counter writeFailureCounter; + private static Timer writeSuccessLatencyTimer; - private volatile Counter updateSuccessCounter; - private volatile Counter updateFailureCounter; - private volatile Timer updateSuccessLatencyTimer; + private static Counter updateSuccessCounter; + private static Counter updateFailureCounter; + private static Timer updateSuccessLatencyTimer; @Override public void init() throws DBException { From c7d042e46c7456c8f463395ea11858af9975ec1e Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Mon, 31 Oct 2022 14:33:02 -0400 Subject: [PATCH 52/72] Update README.md --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5cd7cecfc8..96e63c5926 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,16 @@ # Project -> This repo has been populated by an initial template to help get you started. Please -> make sure to update the content to build a great experience for community-building. +This repository is a fork of the original YCSB repository. We had to create this fork as there was no response to our pull requests from the YCSB maintainers. -As the maintainer of this project, please make a few updates: +This fork contains the following updates to Cosmos DB binding: +- Cosmos DB Java SDK 4.28.0 +- Upgrade to Log4J 2 +- Client diagnostic and exception logging to separate files +- Micrometer integration -- Improving this README.MD file to provide a great experience -- Updating SUPPORT.MD with content about this project's support experience -- Understanding the security reporting process in SECURITY.MD -- Remove this section from the README +Additionally, the “operationcount” type in YCSB core, has been changed to long from int to allow for large “operationcount” value. +We will keep this fork up to date with the upstream YCSB repository and push changes from here, back to the upstream YCSB repository. ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a From 2da5753c42f2a871232efb0e6996847f98019ffa Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Mon, 31 Oct 2022 15:46:22 -0400 Subject: [PATCH 53/72] Update README.md Co-authored-by: Naveen Singh --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96e63c5926..cc3150d608 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This fork contains the following updates to Cosmos DB binding: Additionally, the “operationcount” type in YCSB core, has been changed to long from int to allow for large “operationcount” value. -We will keep this fork up to date with the upstream YCSB repository and push changes from here, back to the upstream YCSB repository. +We will keep this fork up to date with the upstream YCSB repository and push changes from here, back to the upstream YCSB repository if it becomes active again. ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a From 34fbb53be6da13ca27e46d57bb563f0f76a62855 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh Date: Tue, 10 Jan 2023 09:53:25 -0500 Subject: [PATCH 54/72] bumping cosmos version sdk to 4.39.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 245fef0e04..f7a1d5e2a8 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ LICENSE file. 3.1.2 4.4.1 1.8.2 - 4.34.0 + 4.39.0 4.0.0 3.0.0 2.0.1 From 1c10969a82bebf70b81bc17ce54d27d82d36a428 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh Date: Tue, 10 Jan 2023 10:12:51 -0500 Subject: [PATCH 55/72] fixing comment in cosmos client file --- azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index cc161f1cec..886e5aaa94 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -67,7 +67,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** - * Azure Cosmos DB Java SDK 4.28.0 client for YCSB. + * Azure Cosmos DB Java V4 SDK client for YCSB. */ public class AzureCosmosClient extends DB { From 24092d400cb979da0bb8281776701a4c2a552bd7 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh Date: Thu, 12 Jan 2023 14:02:27 -0500 Subject: [PATCH 56/72] Adding cosmos in log diagnotics and exception log folder --- azurecosmos/src/main/resources/log4j2.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index f38cf69409..fa0d31657d 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -2,8 +2,8 @@ %-4r [%t] %-5p %c %x -%m%n - /tmp/cosmos_client_logs/diagnostics - /tmp/cosmos_client_logs/exceptions + /tmp/cosmos_client_logs/cosmos_diagnostics + /tmp/cosmos_client_logs/cosmos_exceptions From c33973be6e14eb2f0380fb73b7b25b46b8fc87d5 Mon Sep 17 00:00:00 2001 From: Naveen Singh Date: Tue, 21 Feb 2023 18:12:28 -0500 Subject: [PATCH 57/72] Bumping cosmos sdk version (#17) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f7a1d5e2a8..aa53fa633e 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ LICENSE file. 3.1.2 4.4.1 1.8.2 - 4.39.0 + 4.41.0 4.0.0 3.0.0 2.0.1 From ff112196509153f20f8fa487f49c2aa5d598d13f Mon Sep 17 00:00:00 2001 From: Naveen Singh Date: Tue, 7 Mar 2023 17:45:56 -0500 Subject: [PATCH 58/72] Creating new reactive stream Mongo driver (#18) * Adding mongo reactive stream * adding mongo stream binding * adding ""mmongodb2" binding * Update pom.xml * Update pom.xml * mongo reactive streams updates * Create README.md * helper * Update MongoDbReactiveStreamsClient.java * Update mongodbreactivestreams.properties * property name change to binding name * Update mongodb.properties * Update MongoDbReactiveStreamsClient.java * Removing explain method * Removing print statement on every read * Bumping mongo reactive stream version * Removed MongoDB2Client reference and added mongodbreactivestreams binding reference * Upgraded to Log4J2 * removed unused logback dependency * This is not required! * Resolving comments * adding back "AsyncMongoDbClient" to script and properties file * checkstyle fix --------- Co-authored-by: Ravi Tella --- bin/bindings.properties | 1 + bin/ycsb | 1 + distribution/pom.xml | 5 + mongodbreactivestreams/README.md | 140 ++++ .../conf/mongodb.properties | 28 + mongodbreactivestreams/pom.xml | 68 ++ .../ycsb/db/MongoDbReactiveStreamsClient.java | 611 ++++++++++++++++++ .../java/site/ycsb/db/OptionsSupport.java | 148 +++++ .../main/java/site/ycsb/db/package-info.java | 26 + .../src/main/resources/log4j2.xml | 19 + .../db/MongoDbReactiveStreamsClientTest.java | 347 ++++++++++ .../java/site/ycsb/db/OptionsSupportTest.java | 184 ++++++ pom.xml | 2 + 13 files changed, 1580 insertions(+) create mode 100644 mongodbreactivestreams/README.md create mode 100644 mongodbreactivestreams/conf/mongodb.properties create mode 100644 mongodbreactivestreams/pom.xml create mode 100644 mongodbreactivestreams/src/main/java/site/ycsb/db/MongoDbReactiveStreamsClient.java create mode 100644 mongodbreactivestreams/src/main/java/site/ycsb/db/OptionsSupport.java create mode 100644 mongodbreactivestreams/src/main/java/site/ycsb/db/package-info.java create mode 100644 mongodbreactivestreams/src/main/resources/log4j2.xml create mode 100644 mongodbreactivestreams/src/test/java/site/ycsb/db/MongoDbReactiveStreamsClientTest.java create mode 100644 mongodbreactivestreams/src/test/java/site/ycsb/db/OptionsSupportTest.java diff --git a/bin/bindings.properties b/bin/bindings.properties index 5c767c7599..a0e9bc10b8 100755 --- a/bin/bindings.properties +++ b/bin/bindings.properties @@ -59,6 +59,7 @@ kudu:site.ycsb.db.KuduYCSBClient memcached:site.ycsb.db.MemcachedClient mongodb:site.ycsb.db.MongoDbClient mongodb-async:site.ycsb.db.AsyncMongoDbClient +mongodbreactivestreams:site.ycsb.db.MongoDbReactiveStreamsClient nosqldb:site.ycsb.db.NoSqlDbClient orientdb:site.ycsb.db.OrientDBClient postgrenosql:site.ycsb.postgrenosql.PostgreNoSQLDBClient diff --git a/bin/ycsb b/bin/ycsb index b5c85e35bb..99922122ee 100755 --- a/bin/ycsb +++ b/bin/ycsb @@ -88,6 +88,7 @@ DATABASES = { "maprjsondb" : "site.ycsb.db.mapr.MapRJSONDBClient", "mongodb" : "site.ycsb.db.MongoDbClient", "mongodb-async": "site.ycsb.db.AsyncMongoDbClient", + "mongodbreactivestreams" : "site.ycsb.db.MongoDbReactiveStreamsClient", "nosqldb" : "site.ycsb.db.NoSqlDbClient", "orientdb" : "site.ycsb.db.OrientDBClient", "postgrenosql" : "site.ycsb.postgrenosql.PostgreNoSQLDBClient", diff --git a/distribution/pom.xml b/distribution/pom.xml index 742a42f511..0165a36495 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -184,6 +184,11 @@ LICENSE file. mongodb-binding ${project.version} + + site.ycsb + mongodbreactivestreams-binding + ${project.version} + site.ycsb nosqldb-binding diff --git a/mongodbreactivestreams/README.md b/mongodbreactivestreams/README.md new file mode 100644 index 0000000000..b60de309e3 --- /dev/null +++ b/mongodbreactivestreams/README.md @@ -0,0 +1,140 @@ + + +## Quick Start + +This section describes how to run YCSB on MongoDB. + +### 1. Start MongoDB + +First, download MongoDB and start `mongod`. For example, to start MongoDB +on x86-64 Linux box: + + wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-x.x.x.tgz + tar xfvz mongodb-linux-x86_64-*.tgz + mkdir /tmp/mongodb + cd mongodb-linux-x86_64-* + ./bin/mongod --dbpath /tmp/mongodb + +Replace x.x.x above with the latest stable release version for MongoDB. +See http://docs.mongodb.org/manual/installation/ for installation steps for various operating systems. + +### 2. Install Java and Maven + +Go to http://www.oracle.com/technetwork/java/javase/downloads/index.html + +and get the url to download the rpm into your server. For example: + + wget http://download.oracle.com/otn-pub/java/jdk/7u40-b43/jdk-7u40-linux-x64.rpm?AuthParam=11232426132 -o jdk-7u40-linux-x64.rpm + rpm -Uvh jdk-7u40-linux-x64.rpm + +Or install via yum/apt-get + + sudo yum install java-devel + +Download MVN from http://maven.apache.org/download.cgi + + wget http://ftp.heanet.ie/mirrors/www.apache.org/dist/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.tar.gz + sudo tar xzf apache-maven-*-bin.tar.gz -C /usr/local + cd /usr/local + sudo ln -s apache-maven-* maven + sudo vi /etc/profile.d/maven.sh + +Add the following to `maven.sh` + + export M2_HOME=/usr/local/maven + export PATH=${M2_HOME}/bin:${PATH} + +Reload bash and test mvn + + bash + mvn -version + +### 3. Set Up YCSB + +Download the YCSB zip file and compile: + + curl -O --location https://github.com/brianfrankcooper/YCSB/releases/download/0.5.0/ycsb-0.5.0.tar.gz + tar xfvz ycsb-0.5.0.tar.gz + cd ycsb-0.5.0 + +### 4. Run YCSB + +Now you are ready to run! First, lets load the data: + + ./bin/ycsb load mongodbreactivestreams -s -P workloads/workloada > outputLoad.txt + +Then, run the workload: + + ./bin/ycsb run mongodbreactivestreams -s -P workloads/workloada > outputRun.txt + +See the next section for the list of configuration parameters for MongoDB. + +## Log Level Control +Due to the mongodb driver defaulting to a log level of DEBUG, the log4j2.xml file is included with this module restricts the org.mongodb logging to WARN. + +## MongoDB Configuration Parameters + +- `mongodb.url` + - This should be a MongoDB URI or connection string. + - See http://docs.mongodb.org/manual/reference/connection-string/ for the standard options. + - Default value is `mongodb://localhost:27017/ycsb?w=1` + - Default value of database is `ycsb` + +- `mongodb.batchsize` + - Useful for the insert workload as it will submit the inserts in batches inproving throughput. + - Default value is `1`. + +- `mongodb.upsert` + - Determines if the insert operation performs an update with the upsert operation or a insert. + Upserts have the advantage that they will continue to work for a partially loaded data set. + - Setting to `true` uses updates, `false` uses insert operations. + - Default value is `false`. + +- `mongodb.writeConcern` + - **Deprecated** - Use the `w` and `journal` options on the MongoDB URI provided by the `mongodb.url`. + - Allowed values are : + - `errors_ignored` + - `unacknowledged` + - `acknowledged` + - `journaled` + - `replica_acknowledged` + - `majority` + - Default value is `acknowledged`. + +- `mongodb.readPreference` + - **Deprecated** - Use the `readPreference` options on the MongoDB URI provided by the `mongodb.url`. + - Allowed values are : + - `primary` + - `primary_preferred` + - `secondary` + - `secondary_preferred` + - `nearest` + - Default value is `primary`. + +- `mongodb.maxconnections` + - **Deprecated** - Use the `maxPoolSize` options on the MongoDB URI provided by the `mongodb.url`. + - Default value is `100`. + +- `mongodb.threadsAllowedToBlockForConnectionMultiplier` + - **Deprecated** - Use the `waitQueueMultiple` options on the MongoDB URI provided by the `mongodb.url`. + - Default value is `5`. + +For example: + + ./bin/ycsb load mongodbreactivestreams -s -P workloads/workloada -p mongodb.url=mongodb://localhost:27017/ycsb?w=0 + diff --git a/mongodbreactivestreams/conf/mongodb.properties b/mongodbreactivestreams/conf/mongodb.properties new file mode 100644 index 0000000000..728bd68943 --- /dev/null +++ b/mongodbreactivestreams/conf/mongodb.properties @@ -0,0 +1,28 @@ +# Copyright (c) 2022 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + +# Mongo DB host uri +# mongodb.url = + +# Set to true to allow inserts to update existing documents. +# If this is false and a document already exists, the insert will fail. +# "true" or "false" +# mongodb.upsert = false Database to be used, if not specified 'ycsb' will be used. + +# Database to be used, if not specified 'ycsb' will be used. +# mongodb.databaseName = ycsb + +# Set insert batchsize, default 1 - to be YCSB-original equivalent +# mongodb.batchsize = 1 diff --git a/mongodbreactivestreams/pom.xml b/mongodbreactivestreams/pom.xml new file mode 100644 index 0000000000..5ea45448e1 --- /dev/null +++ b/mongodbreactivestreams/pom.xml @@ -0,0 +1,68 @@ + + + + + + 4.0.0 + + site.ycsb + binding-parent + 0.18.0-SNAPSHOT + ../binding-parent + + + mongodbreactivestreams-binding + MongoDB ReactiveStreams Binding + jar + + + + org.mongodb + mongodb-driver-reactivestreams + ${mongodbreactivestreams.version} + + + org.apache.logging.log4j + log4j-api + 2.18.0 + + + org.apache.logging.log4j + log4j-core + 2.18.0 + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.18.0 + + + site.ycsb + core + ${project.version} + provided + + + junit + junit + 4.13.1 + test + + + + diff --git a/mongodbreactivestreams/src/main/java/site/ycsb/db/MongoDbReactiveStreamsClient.java b/mongodbreactivestreams/src/main/java/site/ycsb/db/MongoDbReactiveStreamsClient.java new file mode 100644 index 0000000000..54ab7d5b9b --- /dev/null +++ b/mongodbreactivestreams/src/main/java/site/ycsb/db/MongoDbReactiveStreamsClient.java @@ -0,0 +1,611 @@ +/* + * Copyright (c) 2022, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db; + +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.MongoInterruptedException; +import com.mongodb.MongoTimeoutException; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.bulk.BulkWriteResult; +import com.mongodb.client.model.InsertManyOptions; +import com.mongodb.client.model.ReplaceOneModel; +import com.mongodb.client.model.ReplaceOptions; +import com.mongodb.client.result.DeleteResult; +import com.mongodb.client.result.InsertManyResult; +import com.mongodb.client.result.InsertOneResult; +import com.mongodb.client.result.UpdateResult; +import com.mongodb.reactivestreams.client.FindPublisher; +import com.mongodb.reactivestreams.client.MongoClient; +import com.mongodb.reactivestreams.client.MongoClients; +import com.mongodb.reactivestreams.client.MongoCollection; +import com.mongodb.reactivestreams.client.MongoDatabase; +import org.bson.Document; +import org.bson.types.Binary; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import site.ycsb.ByteArrayByteIterator; +import site.ycsb.ByteIterator; +import site.ycsb.DB; +import site.ycsb.DBException; +import site.ycsb.Status; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Vector; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * MongoDB Reactive Streams client for YCSB framework. + */ +public class MongoDbReactiveStreamsClient extends DB { + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbReactiveStreamsClient.class); + private static final String DEFAULT_DATABASE_NAME = "ycsb"; + private static final boolean DEFAULT_USE_UPSERT = false; + private static final int DEFAULT_BATCH_SIZE = 1; + /** + * Used to include a field in a response. + */ + protected static final int INCLUDE = 1; + + /** + * The database to use. + */ + private static String databaseName; + + /** + * The write concern for the requests. + */ + private static final AtomicInteger INIT_COUNT = new AtomicInteger(0); + + /** + * The connection to MongoDB. + */ + private static MongoClient mongoClient; + + /** + * The database to MongoDB. + */ + private static MongoDatabase database; + + /** + * The batch size to use for inserts. + */ + private static int batchSize; + + /** + * If true then use updates with the upsert option for inserts. + */ + private static boolean useUpsert; + + /** + * The default read preference for the test. + */ + private static ReadPreference readPreference; + + /** + * The default write concern for the test. + */ + private static WriteConcern writeConcern; + + // /** The bulk inserts pending for the thread. */ + private final List bulkInserts = new ArrayList(); + + private static final ReplaceOptions REPLACE_WITH_UPSERT = new ReplaceOptions() + .upsert(true); + + private static final InsertManyOptions INSERT_UNORDERED = + new InsertManyOptions().ordered(false); + + /** + * Cleanup any state for this DB. Called once per DB instance; there is one DB + * instance per client thread. + */ + @Override + public final void cleanup() throws DBException { + if (INIT_COUNT.decrementAndGet() == 0) { + try { + mongoClient.close(); + } catch (final Exception e1) { + LOGGER.error("Could not close MongoDB connection pool: " + e1.toString()); + e1.printStackTrace(); + return; + } finally { + mongoClient = null; + database = null; + } + } + } + + /** + * Delete a record from the database. + * + * @param table The name of the table + * @param key The record key of the record to delete. + * @return Zero on success, a non-zero error code on error. See this class's + * description for a discussion of error codes. + */ + @Override + public final Status delete(final String table, final String key) { + try { + final MongoCollection collection = database.getCollection(table); + final Document q = new Document("_id", key); //Builders.start().add("_id", key).build(); + OperationSubscriber deleteSubscriber = new OperationSubscriber(); + collection.withWriteConcern(writeConcern).deleteOne(q).subscribe(deleteSubscriber); + if (deleteSubscriber.first() == null || deleteSubscriber.first().getDeletedCount() <= 0) { + LOGGER.error("Nothing deleted for key " + key); + return Status.NOT_FOUND; + } + return Status.OK; + } catch (final Exception e) { + LOGGER.error(e.toString()); + return Status.ERROR; + } + } + + /** + * Initialize any state for this DB. Called once per DB instance; there is one + * DB instance per client thread. + */ + @Override + public final void init() throws DBException { + INIT_COUNT.incrementAndGet(); + + synchronized (MongoDbReactiveStreamsClient.class) { + if (mongoClient != null) { + return; + } + + // Set insert batchsize, default 1 - to be YCSB-original equivalent + batchSize = this.getIntProperty("mongodb.batchsize", DEFAULT_BATCH_SIZE); + + // Set is inserts are done as upserts. Defaults to false. + useUpsert = this.getBooleanProperty("mongodb.upsert", DEFAULT_USE_UPSERT); + + // Just use the standard connection format URL + // http://docs.mongodb.org/manual/reference/connection-string/ + // to configure the client. + String url = this.getStringProperty("mongodb.url", "mongodb://localhost:27017/ycsb?w=1"); + url = OptionsSupport.updateUrl(url, getProperties()); + + if (!url.startsWith("mongodb://")) { + LOGGER.error("ERROR: Invalid URL: '" + url + + "'. Must be of the form " + + "'mongodb://:,:/database?" + + "options'. See " + + "http://docs.mongodb.org/manual/reference/connection-string/."); + System.exit(1); + } + + ConnectionString connectionString = new ConnectionString(url); + try { + databaseName = connectionString.getDatabase(); + if ((databaseName == null) || databaseName.isEmpty()) { + // Default database is "ycsb" if database is not + // specified in URL + databaseName = this.getStringProperty("mongodb.databaseName", DEFAULT_DATABASE_NAME); + } + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(connectionString) + .retryWrites(false) + .build(); + + mongoClient = MongoClients.create(settings); + + readPreference = settings.getReadPreference(); + writeConcern = settings.getWriteConcern(); + + database = mongoClient.getDatabase(databaseName); + LOGGER.info("mongo connection created with " + url); + } catch (final Exception e1) { + LOGGER.error("Could not initialize MongoDB connection pool for Loader: " + + e1.toString()); + e1.printStackTrace(); + return; + } + } + } + + /** + * Insert a record in the database. Any field/value pairs in the specified + * values HashMap will be written into the record with the specified record + * key. + * + * @param table The name of the table + * @param key The record key of the record to insert. + * @param values A HashMap of field/value pairs to insert in the record + * @return Zero on success, a non-zero error code on error. See the {@link DB} + * class's description for a discussion of error codes. + */ + @Override + public final Status insert(String table, String key, + Map values) { + try { + final MongoCollection collection = database.getCollection(table); + final Document toInsert = new Document("_id", key); + for (final Map.Entry entry : values.entrySet()) { + toInsert.put(entry.getKey(), entry.getValue().toArray()); + } + + OperationSubscriber insertSubscriber; + // Do an upsert. + if (batchSize <= 1) { + long result; + if (useUpsert) { + insertSubscriber = new OperationSubscriber(); + collection.replaceOne(new Document("_id", toInsert.get("_id")), toInsert, REPLACE_WITH_UPSERT) + .subscribe(insertSubscriber); + } else { + insertSubscriber = new OperationSubscriber(); + collection.insertOne(toInsert).subscribe(insertSubscriber); + } + insertSubscriber.await(); + } else { + // Use a bulk insert. + bulkInserts.add(toInsert); + if (bulkInserts.size() == batchSize) { + if (useUpsert) { + List> updates = + new ArrayList>(bulkInserts.size()); + for (Document doc : bulkInserts) { + updates.add(new ReplaceOneModel( + new Document("_id", doc.get("_id")), + doc, REPLACE_WITH_UPSERT)); + } + insertSubscriber = new OperationSubscriber(); + collection.bulkWrite(updates).subscribe(insertSubscriber); + } else { + insertSubscriber = new OperationSubscriber(); + collection.insertMany(bulkInserts, INSERT_UNORDERED).subscribe(insertSubscriber); + } + insertSubscriber.await(); + bulkInserts.clear(); + } else { + return Status.BATCHED_OK; + } + } + return Status.OK; + } catch (final Exception e) { + e.printStackTrace(); + return Status.ERROR; + } + } + + /** + * Read a record from the database. Each field/value pair from the result will + * be stored in a HashMap. + * + * @param table The name of the table + * @param key The record key of the record to read. + * @param fields The list of fields to read, or null for all of them + * @param result A HashMap of field/value pairs for the result + * @return Zero on success, a non-zero error code on error or "not found". + */ + @Override + public final Status read(final String table, final String key, + final Set fields, final Map result) { + try { + MongoCollection collection = database.getCollection(table); + OperationSubscriber readSubscriber = new OperationSubscriber(); + Document query = new Document("_id", key); + FindPublisher findPublisher = collection.find(query); + if (fields != null) { + Document projection = new Document(); + for (String field : fields) { + projection.put(field, INCLUDE); + } + findPublisher.projection(projection); + } + findPublisher.subscribe(readSubscriber); + Document queryResult = readSubscriber.first(); + if (queryResult != null) { + fillMap(result, queryResult); + } + return queryResult != null ? Status.OK : Status.NOT_FOUND; + } catch (final Exception e) { + LOGGER.error(e.toString()); + return Status.ERROR; + } + } + + /** + * Perform a range scan for a set of records in the database. Each field/value + * pair from the result will be stored in a HashMap. + * + * @param table The name of the table + * @param startkey The record key of the first record to read. + * @param recordcount The number of records to read + * @param fields The list of fields to read, or null for all of them + * @param result A Vector of HashMaps, where each HashMap is a set field/value + * pairs for one record + * @return Zero on success, a non-zero error code on error. See the {@link DB} + * class's description for a discussion of error codes. + */ + @Override + public final Status scan(String table, String startkey, + int recordcount, Set fields, + Vector> result) { + try { + MongoCollection collection = database.getCollection(table); + Document scanRange = new Document("$gte", startkey); + Document query = new Document("_id", scanRange); + Document sort = new Document("_id", INCLUDE); + + FindPublisher findPublisher = collection.find(query).sort(sort).limit(1); + if (fields != null) { + Document projection = new Document(); + for (String fieldName : fields) { + projection.put(fieldName, INCLUDE); + } + findPublisher.projection(projection); + } + + result.ensureCapacity(recordcount); + + QuerySubscriber querySubscriber = new QuerySubscriber(result); + findPublisher.subscribe(querySubscriber); + querySubscriber.await(); + + return Status.OK; + } catch (final Exception e) { + LOGGER.error((e.toString())); + return Status.ERROR; + } + } + + /** + * Update a record in the database. Any field/value pairs in the specified + * values HashMap will be written into the record with the specified record + * key, overwriting any existing values with the same field name. + * + * @param table The name of the table + * @param key The record key of the record to write. + * @param values A HashMap of field/value pairs to update in the record + * @return Zero on success, a non-zero error code on error. See the {@link DB} + * class's description for a discussion of error codes. + */ + @Override + public final Status update(String table, String key, + Map values) { + try { + MongoCollection collection = database.getCollection(table); + Document query = new Document("_id", key); + Document fieldsToSet = new Document(); + for (Map.Entry entry : values.entrySet()) { + fieldsToSet.put(entry.getKey(), entry.getValue().toArray()); + } + Document update = new Document("$set", fieldsToSet); + OperationSubscriber updateSubscriber = new OperationSubscriber(); + collection.updateOne(query, update).subscribe(updateSubscriber); + UpdateResult result = updateSubscriber.first(); + if (result.wasAcknowledged() && result.getMatchedCount() == 0) { + LOGGER.error("Nothing updated for key " + key); + return Status.NOT_FOUND; + } + return Status.OK; + } catch (final Exception e) { + LOGGER.error(e.toString()); + return Status.ERROR; + } + } + + /** + * Fills the map with the ByteIterators from the document. + * + * @param result The map to fill. + * @param queryResult The document to fill from. + */ + protected static final void fillMap(final Map result, + final Document queryResult) { + for (Map.Entry entry : queryResult.entrySet()) { + if (entry.getValue() instanceof Binary) { + result.put(entry.getKey(), + new ByteArrayByteIterator(((Binary) entry.getValue()).getData())); + } + } + } + + private String getStringProperty(String propertyName, String defaultValue) { + return getProperties().getProperty(propertyName, defaultValue); + } + + private boolean getBooleanProperty(String propertyName, boolean defaultValue) { + String stringVal = getProperties().getProperty(propertyName, null); + if (stringVal == null) { + return defaultValue; + } + return Boolean.parseBoolean(stringVal); + } + + private int getIntProperty(String propertyName, int defaultValue) { + String stringVal = getProperties().getProperty(propertyName, null); + if (stringVal == null) { + return defaultValue; + } + try { + return Integer.parseInt(stringVal); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + /** + * A Subscriber that stores the publishers results and provides a latch so can block on completion. + * + * @param The publishers result type + */ + public abstract static class ObservableSubscriber implements Subscriber { + private final List received; + private final List errors; + private final CountDownLatch latch; + private volatile Subscription subscription; + private volatile boolean completed; + + /** + * Construct an instance. + */ + public ObservableSubscriber() { + this.received = new ArrayList<>(); + this.errors = new ArrayList<>(); + this.latch = new CountDownLatch(1); + } + + @Override + public void onSubscribe(final Subscription sub) { + this.subscription = sub; + } + + @Override + public void onNext(final T t) { + received.add(t); + } + + @Override + public void onError(final Throwable throwable) { + if (throwable instanceof RuntimeException) { + errors.add((RuntimeException) throwable); + } else { + errors.add(new RuntimeException("Unexpected exception", throwable)); + } + onComplete(); + } + + @Override + public void onComplete() { + completed = true; + latch.countDown(); + } + + /** + * Get received elements. + * + * @return the list of received elements. + */ + public List getReceived() { + return received; + } + + /** + * Get received elements. + * + * @return the list of receive elements + */ + public List get() { + return await().getReceived(); + } + + /** + * Get the first received element. + * + * @return the first received element + */ + public T first() { + List receivedElements = await().getReceived(); + return receivedElements.size() > 0 ? receivedElements.get(0) : null; + } + + /** + * Await completion or error. + * + * @return this + */ + public ObservableSubscriber await() { + return await(300, TimeUnit.SECONDS); + } + + /** + * Await completion or error. + * + * @param timeout how long to wait + * @param unit the time unit + * @return this + */ + public ObservableSubscriber await(final long timeout, final TimeUnit unit) { + subscription.request(Integer.MAX_VALUE); + try { + if (!latch.await(timeout, unit)) { + throw new MongoTimeoutException("Publisher onComplete timed out"); + } + } catch (InterruptedException e) { + throw new MongoInterruptedException("Interrupted waiting for observeration", e); + } + if (!errors.isEmpty()) { + throw errors.get(0); + } + return this; + } + } + + /** + * A CRUD operation Subscriber. + * + * @param The publishers result type + */ + public static class OperationSubscriber extends ObservableSubscriber { + @Override + public void onSubscribe(final Subscription s) { + super.onSubscribe(s); + s.request(Integer.MAX_VALUE); + } + } + + /** + * A Subscriber that prints the json version of each document. + */ + public static class PrintDocumentSubscriber extends OperationSubscriber { + + @Override + public void onNext(final Document document) { + super.onNext(document); + LOGGER.info(document.toJson()); + } + } + + /** + * A Query Subscriber. + */ + public static class QuerySubscriber extends ObservableSubscriber { + private Vector> result; + + public QuerySubscriber(Vector> result) { + this.result = result; + } + + @Override + public void onSubscribe(final Subscription s) { + super.onSubscribe(s); + s.request(Integer.MAX_VALUE); + } + + @Override + public void onNext(final Document t) { + LOGGER.info(t.toJson()); + HashMap resultMap = + new HashMap(); + fillMap(resultMap, t); + result.add(resultMap); + } + } +} diff --git a/mongodbreactivestreams/src/main/java/site/ycsb/db/OptionsSupport.java b/mongodbreactivestreams/src/main/java/site/ycsb/db/OptionsSupport.java new file mode 100644 index 0000000000..619a35017d --- /dev/null +++ b/mongodbreactivestreams/src/main/java/site/ycsb/db/OptionsSupport.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +/** + * OptionsSupport provides methods for handling legacy options. + * + * @author rjm + */ +public final class OptionsSupport { + private static final Logger LOGGER = LoggerFactory.getLogger(OptionsSupport.class); + /** Value for an unavailable property. */ + private static final String UNAVAILABLE = "n/a"; + + /** + * Updates the URL with the appropriate attributes if legacy properties are + * set and the URL does not have the property already set. + * + * @param url + * The URL to update. + * @param props + * The legacy properties. + * @return The updated URL. + */ + public static String updateUrl(String url, Properties props) { + String result = url; + + // max connections. + final String maxConnections = + props.getProperty("mongodb.maxconnections", UNAVAILABLE).toLowerCase(); + if (!UNAVAILABLE.equals(maxConnections)) { + result = addUrlOption(result, "maxPoolSize", maxConnections); + } + + // Blocked thread multiplier. + final String threadsAllowedToBlockForConnectionMultiplier = + props + .getProperty( + "mongodb.threadsAllowedToBlockForConnectionMultiplier", + UNAVAILABLE).toLowerCase(); + if (!UNAVAILABLE.equals(threadsAllowedToBlockForConnectionMultiplier)) { + result = + addUrlOption(result, "waitQueueMultiple", + threadsAllowedToBlockForConnectionMultiplier); + } + + // write concern + String writeConcernType = + props.getProperty("mongodb.writeConcern", UNAVAILABLE).toLowerCase(); + if (!UNAVAILABLE.equals(writeConcernType)) { + if ("errors_ignored".equals(writeConcernType)) { + result = addUrlOption(result, "w", "0"); + } else if ("unacknowledged".equals(writeConcernType)) { + result = addUrlOption(result, "w", "0"); + } else if ("acknowledged".equals(writeConcernType)) { + result = addUrlOption(result, "w", "1"); + } else if ("journaled".equals(writeConcernType)) { + result = addUrlOption(result, "journal", "true"); // this is the + // documented option + // name + result = addUrlOption(result, "j", "true"); // but keep this until + // MongoDB Java driver + // supports "journal" option + } else if ("replica_acknowledged".equals(writeConcernType)) { + result = addUrlOption(result, "w", "2"); + } else if ("majority".equals(writeConcernType)) { + result = addUrlOption(result, "w", "majority"); + } else { + LOGGER.error("WARNING: Invalid writeConcern: '" + + writeConcernType + "' will be ignored. " + + "Must be one of [ unacknowledged | acknowledged | " + + "journaled | replica_acknowledged | majority ]"); + } + } + + // read preference + String readPreferenceType = + props.getProperty("mongodb.readPreference", UNAVAILABLE).toLowerCase(); + if (!UNAVAILABLE.equals(readPreferenceType)) { + if ("primary".equals(readPreferenceType)) { + result = addUrlOption(result, "readPreference", "primary"); + } else if ("primary_preferred".equals(readPreferenceType)) { + result = addUrlOption(result, "readPreference", "primaryPreferred"); + } else if ("secondary".equals(readPreferenceType)) { + result = addUrlOption(result, "readPreference", "secondary"); + } else if ("secondary_preferred".equals(readPreferenceType)) { + result = addUrlOption(result, "readPreference", "secondaryPreferred"); + } else if ("nearest".equals(readPreferenceType)) { + result = addUrlOption(result, "readPreference", "nearest"); + } else { + LOGGER.error("WARNING: Invalid readPreference: '" + + readPreferenceType + "' will be ignored. " + + "Must be one of [ primary | primary_preferred | " + + "secondary | secondary_preferred | nearest ]"); + } + } + + return result; + } + + /** + * Adds an option to the url if it does not already contain the option. + * + * @param url + * The URL to append the options to. + * @param name + * The name of the option. + * @param value + * The value for the option. + * @return The updated URL. + */ + private static String addUrlOption(String url, String name, String value) { + String fullName = name + "="; + if (!url.contains(fullName)) { + if (url.contains("?")) { + return url + "&" + fullName + value; + } + return url + "?" + fullName + value; + } + return url; + } + + /** + * Hidden Constructor. + */ + private OptionsSupport() { + // Nothing. + } +} diff --git a/mongodbreactivestreams/src/main/java/site/ycsb/db/package-info.java b/mongodbreactivestreams/src/main/java/site/ycsb/db/package-info.java new file mode 100644 index 0000000000..3260a4ddab --- /dev/null +++ b/mongodbreactivestreams/src/main/java/site/ycsb/db/package-info.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2022, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/** + * The YCSB binding for MongoDB. + * For additional details on using and configuring the binding see the + * accompanying README.md. + */ +package site.ycsb.db; + diff --git a/mongodbreactivestreams/src/main/resources/log4j2.xml b/mongodbreactivestreams/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..d3dbafd56c --- /dev/null +++ b/mongodbreactivestreams/src/main/resources/log4j2.xml @@ -0,0 +1,19 @@ + + + + %-4r [%t] %-5p %c %x -%m%n + + + + + + + + + + + + + + + diff --git a/mongodbreactivestreams/src/test/java/site/ycsb/db/MongoDbReactiveStreamsClientTest.java b/mongodbreactivestreams/src/test/java/site/ycsb/db/MongoDbReactiveStreamsClientTest.java new file mode 100644 index 0000000000..62ede28522 --- /dev/null +++ b/mongodbreactivestreams/src/test/java/site/ycsb/db/MongoDbReactiveStreamsClientTest.java @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2022, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db; + +import org.junit.BeforeClass; +import org.junit.Test; +import site.ycsb.ByteArrayByteIterator; +import site.ycsb.ByteIterator; +import site.ycsb.DB; +import site.ycsb.Status; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.Vector; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeNoException; + +/** + * MongoDbReactiveStreamsClientTest provides runs the basic DB test cases. + *

+ * The tests will be skipped if MongoDB is not running on port 27017 on the + * local machine. See the README.md for how to get MongoDB running. + *

+ */ +public class MongoDbReactiveStreamsClientTest { + + /** The default port for MongoDB. */ + private static final int MONGODB_DEFAULT_PORT = 27017; + + /** The client to use. */ + private DB myClient = null; + + /** + * Verifies the mongod process (or some process) is running on port 27017, if + * not the tests are skipped. + */ + @BeforeClass + public static void setUpBeforeClass() { + // Test if we can connect. + Socket socket = null; + try { + // Connect + socket = new Socket(InetAddress.getLocalHost(), MONGODB_DEFAULT_PORT); + assertThat("Socket is not bound.", socket.getLocalPort(), not(-1)); + } catch (IOException connectFailed) { + assumeNoException("MongoDB is not running. Skipping tests.", + connectFailed); + } finally { + if (socket != null) { + try { + socket.close(); + } catch (IOException ignore) { + // Ignore. + } + } + socket = null; + } + } + + /** + * Test method for {@link DB#insert}, {@link DB#read}, and {@link DB#delete} . + */ + @Test + public void testInsertReadDelete() { + final DB client = getDB(); + + final String table = getClass().getSimpleName(); + final String id = "delete"; + + HashMap inserted = + new HashMap(); + inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); + Status result = client.insert(table, id, inserted); + assertThat("Insert did not return success (0).", result, is(Status.OK)); + + HashMap read = new HashMap(); + Set keys = Collections.singleton("a"); + result = client.read(table, id, keys, read); + assertThat("Read did not return success (0).", result, is(Status.OK)); + for (String key : keys) { + ByteIterator iter = read.get(key); + + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); + assertFalse(iter.hasNext()); + } + + result = client.delete(table, id); + assertThat("Delete did not return success (0).", result, is(Status.OK)); + + read.clear(); + result = client.read(table, id, null, read); + assertThat("Read, after delete, did not return not found (1).", result, + is(Status.NOT_FOUND)); + assertThat("Found the deleted fields.", read.size(), is(0)); + + result = client.delete(table, id); + assertThat("Delete did not return not found (1).", result, is(Status.NOT_FOUND)); + } + + /** + * Test method for {@link DB#insert}, {@link DB#read}, and {@link DB#update} . + */ + @Test + public void testInsertReadUpdate() { + DB client = getDB(); + + final String table = getClass().getSimpleName(); + final String id = "update"; + + HashMap inserted = + new HashMap(); + inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); + Status result = client.insert(table, id, inserted); + assertThat("Insert did not return success (0).", result, is(Status.OK)); + + HashMap read = new HashMap(); + Set keys = Collections.singleton("a"); + result = client.read(table, id, keys, read); + assertThat("Read did not return success (0).", result, is(Status.OK)); + for (String key : keys) { + ByteIterator iter = read.get(key); + + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); + assertFalse(iter.hasNext()); + } + + HashMap updated = new HashMap(); + updated.put("a", new ByteArrayByteIterator(new byte[] { 5, 6, 7, 8 })); + result = client.update(table, id, updated); + assertThat("Update did not return success (0).", result, is(Status.OK)); + + read.clear(); + result = client.read(table, id, null, read); + assertThat("Read, after update, did not return success (0).", result, is(Status.OK)); + for (String key : keys) { + ByteIterator iter = read.get(key); + + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 5))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 6))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 7))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 8))); + assertFalse(iter.hasNext()); + } + } + + /** + * Test method for {@link DB#insert}, {@link DB#read}, and {@link DB#update} . + */ + @Test + public void testInsertReadUpdateWithUpsert() { + Properties props = new Properties(); + props.setProperty("mongodb.upsert", "true"); + DB client = getDB(props); + + final String table = getClass().getSimpleName(); + final String id = "updateWithUpsert"; + + HashMap inserted = + new HashMap(); + inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); + Status result = client.insert(table, id, inserted); + assertThat("Insert did not return success (0).", result, is(Status.OK)); + + HashMap read = new HashMap(); + Set keys = Collections.singleton("a"); + result = client.read(table, id, keys, read); + assertThat("Read did not return success (0).", result, is(Status.OK)); + for (String key : keys) { + ByteIterator iter = read.get(key); + + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); + assertFalse(iter.hasNext()); + } + + HashMap updated = new HashMap(); + updated.put("a", new ByteArrayByteIterator(new byte[] { 5, 6, 7, 8 })); + result = client.update(table, id, updated); + assertThat("Update did not return success (0).", result, is(Status.OK)); + + read.clear(); + result = client.read(table, id, null, read); + assertThat("Read, after update, did not return success (0).", result, is(Status.OK)); + for (String key : keys) { + ByteIterator iter = read.get(key); + + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 5))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 6))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 7))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 8))); + assertFalse(iter.hasNext()); + } + } + + /** + * Test method for {@link DB#scan}. + */ + @Test + public void testScan() { + final DB client = getDB(); + + final String table = getClass().getSimpleName(); + + // Insert a bunch of documents. + for (int i = 0; i < 100; ++i) { + HashMap inserted = + new HashMap(); + inserted.put("a", new ByteArrayByteIterator(new byte[] { + (byte) (i & 0xFF), (byte) (i >> 8 & 0xFF), (byte) (i >> 16 & 0xFF), + (byte) (i >> 24 & 0xFF) })); + Status result = client.insert(table, padded(i), inserted); + assertThat("Insert did not return success (0).", result, is(Status.OK)); + } + + Set keys = Collections.singleton("a"); + Vector> results = + new Vector>(); + Status result = client.scan(table, "00050", 5, null, results); + assertThat("Read did not return success (0).", result, is(Status.OK)); + assertThat(results.size(), is(5)); + for (int i = 0; i < 5; ++i) { + Map read = results.get(i); + for (String key : keys) { + ByteIterator iter = read.get(key); + + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) ((i + 50) & 0xFF)))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), + is(Byte.valueOf((byte) ((i + 50) >> 8 & 0xFF)))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), + is(Byte.valueOf((byte) ((i + 50) >> 16 & 0xFF)))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), + is(Byte.valueOf((byte) ((i + 50) >> 24 & 0xFF)))); + assertFalse(iter.hasNext()); + } + } + } + + /** + * Gets the test DB. + * + * @return The test DB. + */ + DB getDB() { + return getDB(new Properties()); + } + + DB getDB(Properties props) { + if( myClient == null ) { + myClient = instantiateClient(); + myClient.setProperties(props); + try { + myClient.init(); + } catch (Exception error) { + assumeNoException(error); + } + } + return myClient; + } + /** + * Creates a zero padded integer. + * + * @param i + * The integer to padd. + * @return The padded integer. + */ + private String padded(int i) { + String result = String.valueOf(i); + while (result.length() < 5) { + result = "0" + result; + } + return result; + } + + DB instantiateClient() { + return new MongoDbReactiveStreamsClient(); + } +} \ No newline at end of file diff --git a/mongodbreactivestreams/src/test/java/site/ycsb/db/OptionsSupportTest.java b/mongodbreactivestreams/src/test/java/site/ycsb/db/OptionsSupportTest.java new file mode 100644 index 0000000000..3a4d08d015 --- /dev/null +++ b/mongodbreactivestreams/src/test/java/site/ycsb/db/OptionsSupportTest.java @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db; + +import org.junit.Test; + +import java.util.Properties; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static site.ycsb.db.OptionsSupport.updateUrl; + +/** + * OptionsSupportTest provides tests for the OptionsSupport class. + * + * @author rjm + */ +public class OptionsSupportTest { + + /** + * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for + * {@code mongodb.maxconnections}. + */ + @Test + public void testUpdateUrlMaxConnections() { + assertThat( + updateUrl("mongodb://locahost:27017/", + props("mongodb.maxconnections", "1234")), + is("mongodb://locahost:27017/?maxPoolSize=1234")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.maxconnections", "1234")), + is("mongodb://locahost:27017/?foo=bar&maxPoolSize=1234")); + assertThat( + updateUrl("mongodb://locahost:27017/?maxPoolSize=1", + props("mongodb.maxconnections", "1234")), + is("mongodb://locahost:27017/?maxPoolSize=1")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), + is("mongodb://locahost:27017/?foo=bar")); + } + + /** + * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for + * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. + */ + @Test + public void testUpdateUrlWaitQueueMultiple() { + assertThat( + updateUrl( + "mongodb://locahost:27017/", + props("mongodb.threadsAllowedToBlockForConnectionMultiplier", + "1234")), + is("mongodb://locahost:27017/?waitQueueMultiple=1234")); + assertThat( + updateUrl( + "mongodb://locahost:27017/?foo=bar", + props("mongodb.threadsAllowedToBlockForConnectionMultiplier", + "1234")), + is("mongodb://locahost:27017/?foo=bar&waitQueueMultiple=1234")); + assertThat( + updateUrl( + "mongodb://locahost:27017/?waitQueueMultiple=1", + props("mongodb.threadsAllowedToBlockForConnectionMultiplier", + "1234")), is("mongodb://locahost:27017/?waitQueueMultiple=1")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), + is("mongodb://locahost:27017/?foo=bar")); + } + + /** + * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for + * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. + */ + @Test + public void testUpdateUrlWriteConcern() { + assertThat( + updateUrl("mongodb://locahost:27017/", + props("mongodb.writeConcern", "errors_ignored")), + is("mongodb://locahost:27017/?w=0")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.writeConcern", "unacknowledged")), + is("mongodb://locahost:27017/?foo=bar&w=0")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.writeConcern", "acknowledged")), + is("mongodb://locahost:27017/?foo=bar&w=1")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.writeConcern", "journaled")), + is("mongodb://locahost:27017/?foo=bar&journal=true&j=true")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.writeConcern", "replica_acknowledged")), + is("mongodb://locahost:27017/?foo=bar&w=2")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.writeConcern", "majority")), + is("mongodb://locahost:27017/?foo=bar&w=majority")); + + // w already exists. + assertThat( + updateUrl("mongodb://locahost:27017/?w=1", + props("mongodb.writeConcern", "acknowledged")), + is("mongodb://locahost:27017/?w=1")); + + // Unknown options + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), + is("mongodb://locahost:27017/?foo=bar")); + } + + /** + * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for + * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. + */ + @Test + public void testUpdateUrlReadPreference() { + assertThat( + updateUrl("mongodb://locahost:27017/", + props("mongodb.readPreference", "primary")), + is("mongodb://locahost:27017/?readPreference=primary")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.readPreference", "primary_preferred")), + is("mongodb://locahost:27017/?foo=bar&readPreference=primaryPreferred")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.readPreference", "secondary")), + is("mongodb://locahost:27017/?foo=bar&readPreference=secondary")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.readPreference", "secondary_preferred")), + is("mongodb://locahost:27017/?foo=bar&readPreference=secondaryPreferred")); + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", + props("mongodb.readPreference", "nearest")), + is("mongodb://locahost:27017/?foo=bar&readPreference=nearest")); + + // readPreference already exists. + assertThat( + updateUrl("mongodb://locahost:27017/?readPreference=primary", + props("mongodb.readPreference", "secondary")), + is("mongodb://locahost:27017/?readPreference=primary")); + + // Unknown options + assertThat( + updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), + is("mongodb://locahost:27017/?foo=bar")); + } + + /** + * Factory method for a {@link Properties} object. + * + * @param key + * The key for the property to set. + * @param value + * The value for the property to set. + * @return The {@link Properties} with the property added. + */ + private Properties props(String key, String value) { + Properties props = new Properties(); + + props.setProperty(key, value); + + return props; + } + +} diff --git a/pom.xml b/pom.xml index aa53fa633e..ca46c4454a 100644 --- a/pom.xml +++ b/pom.xml @@ -135,6 +135,7 @@ LICENSE file. 1.1.8-mapr-1710 3.11.0 2.0.1 + 4.9.0 2.1.1 2.2.37 UTF-8 @@ -187,6 +188,7 @@ LICENSE file. maprjsondb memcached mongodb + mongodbreactivestreams nosqldb orientdb postgrenosql From a4a882269b660263d82de44769694cb04a916c66 Mon Sep 17 00:00:00 2001 From: Naveen Singh Date: Tue, 9 May 2023 10:30:02 -0400 Subject: [PATCH 59/72] Adding preferred region list config to Azure Cosmos client (#21) * Adding preferred region capablity * Adding string isempty instead of null * Fixing formating * Fixing formatting --- azurecosmos/conf/azurecosmos.properties | 4 ++++ .../main/java/site/ycsb/db/AzureCosmosClient.java | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/azurecosmos/conf/azurecosmos.properties b/azurecosmos/conf/azurecosmos.properties index 2b4b42e18d..afadfd9ea7 100644 --- a/azurecosmos/conf/azurecosmos.properties +++ b/azurecosmos/conf/azurecosmos.properties @@ -92,3 +92,7 @@ # application insight connection String # azurecosmos.appInsightConnectionString = + +# Sets the comma separated preferred region list. +# Default value is null. +# azurecosmos.preferredRegionList = diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 886e5aaa94..d21dc8c967 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -25,6 +25,7 @@ import com.azure.cosmos.DirectConnectionConfig; import com.azure.cosmos.GatewayConnectionConfig; import com.azure.cosmos.ThrottlingRetryOptions; +import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; import com.azure.cosmos.models.CosmosItemRequestOptions; import com.azure.cosmos.models.CosmosItemResponse; import com.azure.cosmos.models.CosmosPatchOperations; @@ -55,6 +56,7 @@ import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -224,6 +226,13 @@ private void initAzureCosmosClient() throws DBException { gatewayConnectionConfig.setIdleConnectionTimeout(Duration.ofSeconds(gatewayIdleConnectionTimeoutInSeconds)); } + String preferredRegions = this.getStringProperty("azurecosmos.preferredRegionList", null); + List preferredRegionList = null; + if (StringUtils.isNotEmpty(preferredRegions)) { + preferredRegions = preferredRegions.trim(); + preferredRegionList = new ArrayList<>(Arrays.asList(preferredRegions.split(","))); + } + try { LOGGER.info( "Creating Cosmos DB client {}, useGateway={}, consistencyLevel={}," @@ -243,6 +252,10 @@ private void initAzureCosmosClient() throws DBException { builder = builder.directMode(directConnectionConfig); } + if (preferredRegionList != null && preferredRegionList.size() > 0) { + builder.preferredRegions(preferredRegionList); + } + AzureCosmosClient.client = builder.buildClient(); LOGGER.info("Azure Cosmos DB connection created to {}", uri); } catch (IllegalArgumentException e) { From 873a278a24c8edbf3b020dc94b7218a5f026947d Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Thu, 15 Jun 2023 14:54:50 -0400 Subject: [PATCH 60/72] fix for log configuration for delete workload (#22) --- .../src/main/java/site/ycsb/db/AzureCosmosClient.java | 2 +- azurecosmos/src/main/resources/log4j2.xml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index d21dc8c967..22306e682d 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -97,7 +97,7 @@ public class AzureCosmosClient extends DB { private static final Marker CREATE_EXCEPTION = MarkerFactory.getMarker("CREATE_EXCEPTION"); private static final Marker READ_EXCEPTION = MarkerFactory.getMarker("READ_EXCEPTION"); private static final Marker PATCH_EXCEPTION = MarkerFactory.getMarker("PATCH_EXCEPTION"); - private static final Marker DELETE_EXCEPTION = MarkerFactory.getMarker("DELETE_DIAGNOSTIC"); + private static final Marker DELETE_EXCEPTION = MarkerFactory.getMarker("DELETE_EXCEPTION"); private static final Marker QUERY_EXCEPTION = MarkerFactory.getMarker("QUERY_EXCEPTION"); diff --git a/azurecosmos/src/main/resources/log4j2.xml b/azurecosmos/src/main/resources/log4j2.xml index fa0d31657d..5f5766503a 100644 --- a/azurecosmos/src/main/resources/log4j2.xml +++ b/azurecosmos/src/main/resources/log4j2.xml @@ -14,6 +14,7 @@ + @@ -79,7 +80,7 @@ filePattern="${BASE_PATH_DIAGNOSTICS}/delete/diagnostics.%d{yyyy-MM-dd-hh-mm-ss}.log.gz" ignoreExceptions="false"> - + @@ -144,7 +145,7 @@ filePattern="${BASE_PATH_EXCEPTION}/delete/exceptions.%d{yyyy-MM-dd-hh-mm-ss}.log.gz" ignoreExceptions="false"> - + From cea1da2d8d70601cafaf9d87080404e52fa29424 Mon Sep 17 00:00:00 2001 From: Martijn Verburg Date: Sat, 15 Jul 2023 12:28:27 +1200 Subject: [PATCH 61/72] Add GH Action Workflow --- .github/workflows/pr-builder.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/pr-builder.yml diff --git a/.github/workflows/pr-builder.yml b/.github/workflows/pr-builder.yml new file mode 100644 index 0000000000..4077e50564 --- /dev/null +++ b/.github/workflows/pr-builder.yml @@ -0,0 +1,27 @@ +name: YCSB build with Maven Wrapper + +on: + pull_request: + branches: [ main ] + paths-ignore: + - '**.md' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + java: [11, 17] + + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 # 3.11.0 + with: + java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'maven' + + - name: Build with Maven Wrapper + run: ./mvnw clean install site \ No newline at end of file From 60613e97a25bc7586ad4e0badc8f37452e673f63 Mon Sep 17 00:00:00 2001 From: Ravi Tella Date: Wed, 6 Dec 2023 23:03:52 -0500 Subject: [PATCH 62/72] Cosmos SDK upgraded to 4.52.0 (#36) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ca46c4454a..6a2b52f13a 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ LICENSE file. 3.1.2 4.4.1 1.8.2 - 4.41.0 + 4.52.0 4.0.0 3.0.0 2.0.1 From 6c9d4f1d155729babe353db39ad5474eaaf76812 Mon Sep 17 00:00:00 2001 From: Darshan Patnekar Date: Mon, 18 Mar 2024 13:00:53 -0700 Subject: [PATCH 63/72] Changed micrometer implementation to send info to Open Telemetry Changed micrometer implementation to send latency metrics and cosmos client diagnostics to Open Telemetry. --- azurecosmos/pom.xml | 4 +- .../java/site/ycsb/db/AzureCosmosClient.java | 101 ++++++------------ 2 files changed, 33 insertions(+), 72 deletions(-) diff --git a/azurecosmos/pom.xml b/azurecosmos/pom.xml index 5da05181b3..6e997c8b49 100644 --- a/azurecosmos/pom.xml +++ b/azurecosmos/pom.xml @@ -58,8 +58,8 @@ LICENSE file. io.micrometer - micrometer-registry-azure-monitor - 1.9.2 + micrometer-core + 1.6.1 diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 22306e682d..28fdca49f3 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -16,33 +16,15 @@ package site.ycsb.db; -import com.azure.cosmos.ConsistencyLevel; -import com.azure.cosmos.CosmosClient; -import com.azure.cosmos.CosmosClientBuilder; -import com.azure.cosmos.CosmosContainer; -import com.azure.cosmos.CosmosDatabase; -import com.azure.cosmos.CosmosException; -import com.azure.cosmos.DirectConnectionConfig; -import com.azure.cosmos.GatewayConnectionConfig; -import com.azure.cosmos.ThrottlingRetryOptions; +import com.azure.cosmos.*; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.models.CosmosItemRequestOptions; -import com.azure.cosmos.models.CosmosItemResponse; -import com.azure.cosmos.models.CosmosPatchOperations; -import com.azure.cosmos.models.CosmosQueryRequestOptions; -import com.azure.cosmos.models.FeedResponse; -import com.azure.cosmos.models.PartitionKey; -import com.azure.cosmos.models.SqlParameter; -import com.azure.cosmos.models.SqlQuerySpec; +import com.azure.cosmos.models.*; import com.azure.cosmos.util.CosmosPagedIterable; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.microsoft.applicationinsights.TelemetryConfiguration; -import io.micrometer.azuremonitor.AzureMonitorConfig; -import io.micrometer.azuremonitor.AzureMonitorMeterRegistry; -import io.micrometer.core.instrument.Clock; import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Timer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -118,8 +100,6 @@ public class AzureCosmosClient extends DB { private static boolean includeExceptionStackInLog; private static Map containerCache; private static String userAgent; - - private static AzureMonitorMeterRegistry azureMonitorMeterRegistry; private static Counter readSuccessCounter; private static Counter readFailureCounter; @@ -162,7 +142,7 @@ private void initAzureCosmosClient() throws DBException { } String uri = this.getStringProperty("azurecosmos.uri", null); - if (primaryKey == null || primaryKey.isEmpty()) { + if (uri == null || uri.isEmpty()) { throw new DBException("Missing uri required to connect to the database."); } @@ -243,8 +223,18 @@ private void initAzureCosmosClient() throws DBException { AzureCosmosClient.maxDegreeOfParallelism, AzureCosmosClient.maxBufferedItemCount, AzureCosmosClient.preferredPageSize); - CosmosClientBuilder builder = new CosmosClientBuilder().endpoint(uri).key(primaryKey) - .throttlingRetryOptions(retryOptions).consistencyLevel(consistencyLevel).userAgentSuffix(userAgent); + CosmosClientBuilder builder = new CosmosClientBuilder() + .endpoint(uri) + .key(primaryKey) + .throttlingRetryOptions(retryOptions) + .consistencyLevel(consistencyLevel) + .userAgentSuffix(userAgent) + .clientTelemetryConfig(new CosmosClientTelemetryConfig() + .diagnosticsThresholds( + new CosmosDiagnosticsThresholds() + .setPointOperationLatencyThreshold(Duration.ofMillis(100)) + .setNonPointOperationLatencyThreshold(Duration.ofMillis(500)) + .setRequestChargeThreshold(100))); if (useGateway) { builder = builder.gatewayMode(gatewayConnectionConfig); @@ -281,7 +271,7 @@ private void initAzureCosmosClient() throws DBException { String appInsightConnectionString = this.getStringProperty("azurecosmos.appInsightConnectionString", null); if (appInsightConnectionString != null) { - this.azureMonitorMeterRegistry = this.azureMonitorMeterRegistry(appInsightConnectionString); + System.setProperty("applicationinsights.connection.string", appInsightConnectionString); registerMeter(); } } @@ -646,72 +636,43 @@ private String createSelectTop(Set fields, int top) { } } - private synchronized AzureMonitorMeterRegistry azureMonitorMeterRegistry(String appInsightConnectionString) { - if (this.azureMonitorMeterRegistry == null) { - Duration step = Duration.ofSeconds(Integer.getInteger("azure.cosmos.monitoring.azureMonitor.step", 10)); - boolean enabled = !Boolean.getBoolean("azure.cosmos.monitoring.azureMonitor.disabled"); - final AzureMonitorConfig config = new AzureMonitorConfig() { - @Override - public String get(String key) { - return null; - } - - @Override - public Duration step() { - return step; - } - - @Override - public boolean enabled() { - return enabled; - } - }; - TelemetryConfiguration telemetryConfiguration = TelemetryConfiguration.createDefault(); - telemetryConfiguration.setConnectionString(appInsightConnectionString); - azureMonitorMeterRegistry = AzureMonitorMeterRegistry - .builder(config) - .clock(Clock.SYSTEM) - .telemetryConfiguration(telemetryConfiguration) - .build(); - } - return this.azureMonitorMeterRegistry; - } - private void registerMeter() { + System.setProperty("applicationinsights.metrics.interval.seconds", "10"); + if (this.getDoubleProperty("readproportion", 0) > 0) { - readSuccessCounter = this.azureMonitorMeterRegistry.counter("Read Successful Operations"); - readFailureCounter = this.azureMonitorMeterRegistry.counter("Read Unsuccessful Operations"); + readSuccessCounter = Metrics.globalRegistry.counter("Read Successful Operations"); + readFailureCounter = Metrics.globalRegistry.counter("Read Unsuccessful Operations"); readSuccessLatencyTimer = Timer.builder("Read Successful Latency") .publishPercentiles(0.5, 0.95, 0.99, 0.999, 0.9999) .publishPercentileHistogram() - .register(this.azureMonitorMeterRegistry); + .register(Metrics.globalRegistry); } if (this.getDoubleProperty("insertproportion", 0) > 0) { - writeSuccessCounter = this.azureMonitorMeterRegistry.counter("Write Successful Operations"); - writeFailureCounter = this.azureMonitorMeterRegistry.counter("Write Unsuccessful Operations"); + writeSuccessCounter = Metrics.globalRegistry.counter("Write Successful Operations"); + writeFailureCounter = Metrics.globalRegistry.counter("Write Unsuccessful Operations"); writeSuccessLatencyTimer = Timer.builder("Write Successful Latency") .publishPercentiles(0.5, 0.95, 0.99, 0.999, 0.9999) .publishPercentileHistogram() - .register(this.azureMonitorMeterRegistry); + .register(Metrics.globalRegistry); } if (this.getDoubleProperty("scanproportion", 0) > 0) { - scanSuccessCounter = this.azureMonitorMeterRegistry.counter("Scan Successful Operations"); - scanFailureCounter = this.azureMonitorMeterRegistry.counter("Scan Unsuccessful Operations"); + scanSuccessCounter = Metrics.globalRegistry.counter("Scan Successful Operations"); + scanFailureCounter = Metrics.globalRegistry.counter("Scan Unsuccessful Operations"); scanSuccessLatencyTimer = Timer.builder("Scan Successful Latency") .publishPercentiles(0.5, 0.95, 0.99, 0.999, 0.9999) .publishPercentileHistogram() - .register(this.azureMonitorMeterRegistry); + .register(Metrics.globalRegistry); } if (this.getDoubleProperty("updateproportion", 0) > 0) { - updateSuccessCounter = this.azureMonitorMeterRegistry.counter("Update Successful Operations"); - updateFailureCounter = this.azureMonitorMeterRegistry.counter("Update Unsuccessful Operations"); + updateSuccessCounter = Metrics.globalRegistry.counter("Update Successful Operations"); + updateFailureCounter = Metrics.globalRegistry.counter("Update Unsuccessful Operations"); updateSuccessLatencyTimer = Timer.builder("Update Successful Latency") .publishPercentiles(0.5, 0.95, 0.99, 0.999, 0.9999) .publishPercentileHistogram() - .register(this.azureMonitorMeterRegistry); + .register(Metrics.globalRegistry); } } } \ No newline at end of file From 0e30cc50655cdd4986889878ab93c4cb7c14b518 Mon Sep 17 00:00:00 2001 From: Darshan Patnekar Date: Mon, 18 Mar 2024 13:24:42 -0700 Subject: [PATCH 64/72] Replaced * with single imports --- .../java/site/ycsb/db/AzureCosmosClient.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 28fdca49f3..5b0594f7a8 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -16,9 +16,26 @@ package site.ycsb.db; -import com.azure.cosmos.*; +import com.azure.cosmos.ConsistencyLevel; +import com.azure.cosmos.CosmosClient; +import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.CosmosContainer; +import com.azure.cosmos.CosmosDatabase; +import com.azure.cosmos.CosmosDiagnosticsThresholds; +import com.azure.cosmos.CosmosException; +import com.azure.cosmos.DirectConnectionConfig; +import com.azure.cosmos.GatewayConnectionConfig; +import com.azure.cosmos.ThrottlingRetryOptions; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.models.*; +import com.azure.cosmos.models.CosmosClientTelemetryConfig; +import com.azure.cosmos.models.CosmosItemRequestOptions; +import com.azure.cosmos.models.CosmosItemResponse; +import com.azure.cosmos.models.CosmosPatchOperations; +import com.azure.cosmos.models.CosmosQueryRequestOptions; +import com.azure.cosmos.models.FeedResponse; +import com.azure.cosmos.models.PartitionKey; +import com.azure.cosmos.models.SqlParameter; +import com.azure.cosmos.models.SqlQuerySpec; import com.azure.cosmos.util.CosmosPagedIterable; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; From a1ecd2c96163ad6bf35f299e938759059d01363b Mon Sep 17 00:00:00 2001 From: Darshan Patnekar Date: Tue, 19 Mar 2024 14:47:59 -0700 Subject: [PATCH 65/72] parameterized cosmos client diagnostics log options --- azurecosmos/conf/azurecosmos.properties | 5 +++++ azurecosmos/pom.xml | 5 +++++ .../java/site/ycsb/db/AzureCosmosClient.java | 19 ++++++++++++------- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/azurecosmos/conf/azurecosmos.properties b/azurecosmos/conf/azurecosmos.properties index afadfd9ea7..2b5e8f140c 100644 --- a/azurecosmos/conf/azurecosmos.properties +++ b/azurecosmos/conf/azurecosmos.properties @@ -93,6 +93,11 @@ # application insight connection String # azurecosmos.appInsightConnectionString = +# Cosmos Client Diagnostics logs options +# azurecosmos.pointOperationLatencyThresholdInMS = 100 +# azurecosmos.nonPointOperationLatencyThresholdInMS = 500 +# azurecosmos.requestChargeThreshold = 100 + # Sets the comma separated preferred region list. # Default value is null. # azurecosmos.preferredRegionList = diff --git a/azurecosmos/pom.xml b/azurecosmos/pom.xml index 6e997c8b49..155eea5cf6 100644 --- a/azurecosmos/pom.xml +++ b/azurecosmos/pom.xml @@ -61,5 +61,10 @@ LICENSE file. micrometer-core 1.6.1 + + com.microsoft.azure + applicationinsights-agent + 3.5.1 + diff --git a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java index 5b0594f7a8..37355f603d 100644 --- a/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java +++ b/azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java @@ -229,7 +229,15 @@ private void initAzureCosmosClient() throws DBException { preferredRegions = preferredRegions.trim(); preferredRegionList = new ArrayList<>(Arrays.asList(preferredRegions.split(","))); } - + + int pointOperationLatencyThresholdInMS = this.getIntProperty("azurecosmos.pointOperationLatencyThresholdInMS", + 100); + + int nonPointOperationLatencyThresholdInMS = this.getIntProperty("azurecosmos.nonPointOperationLatencyThresholdInMS", + 500); + + int requestChargeThreshold = this.getIntProperty("azurecosmos.requestChargeThreshold", 100); + try { LOGGER.info( "Creating Cosmos DB client {}, useGateway={}, consistencyLevel={}," @@ -249,9 +257,9 @@ private void initAzureCosmosClient() throws DBException { .clientTelemetryConfig(new CosmosClientTelemetryConfig() .diagnosticsThresholds( new CosmosDiagnosticsThresholds() - .setPointOperationLatencyThreshold(Duration.ofMillis(100)) - .setNonPointOperationLatencyThreshold(Duration.ofMillis(500)) - .setRequestChargeThreshold(100))); + .setPointOperationLatencyThreshold(Duration.ofMillis(pointOperationLatencyThresholdInMS)) + .setNonPointOperationLatencyThreshold(Duration.ofMillis(nonPointOperationLatencyThresholdInMS)) + .setRequestChargeThreshold(requestChargeThreshold))); if (useGateway) { builder = builder.gatewayMode(gatewayConnectionConfig); @@ -288,7 +296,6 @@ private void initAzureCosmosClient() throws DBException { String appInsightConnectionString = this.getStringProperty("azurecosmos.appInsightConnectionString", null); if (appInsightConnectionString != null) { - System.setProperty("applicationinsights.connection.string", appInsightConnectionString); registerMeter(); } } @@ -654,8 +661,6 @@ private String createSelectTop(Set fields, int top) { } private void registerMeter() { - System.setProperty("applicationinsights.metrics.interval.seconds", "10"); - if (this.getDoubleProperty("readproportion", 0) > 0) { readSuccessCounter = Metrics.globalRegistry.counter("Read Successful Operations"); readFailureCounter = Metrics.globalRegistry.counter("Read Unsuccessful Operations"); From 49def82e07496861be9339be955a87766f0fb5fc Mon Sep 17 00:00:00 2001 From: Darshan Patnekar Date: Wed, 20 Mar 2024 00:45:21 -0700 Subject: [PATCH 66/72] updated ycsb added jvm args --- bin/ycsb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ycsb b/bin/ycsb index 99922122ee..f5b5227384 100755 --- a/bin/ycsb +++ b/bin/ycsb @@ -237,7 +237,7 @@ def main(): p.add_argument('-cp', dest='classpath', help="""Additional classpath entries, e.g. '-cp /tmp/hbase-1.0.1.1/conf'. Will be prepended to the YCSB classpath.""") - p.add_argument("-jvm-args", default=[], type=shlex.split, + p.add_argument("-jvm-args", default="-javaagent:/tmp/ycsb/ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/lib/applicationinsights-agent-3.5.1.jar", type=shlex.split, help="""Additional arguments to pass to 'java', e.g. '-Xmx4g'""") p.add_argument("command", choices=sorted(COMMANDS), From f55a279a0665dd08a9bd1ff2a13d880fc7c73569 Mon Sep 17 00:00:00 2001 From: Darshan Patnekar Date: Wed, 20 Mar 2024 10:25:17 -0700 Subject: [PATCH 67/72] removed default args --- bin/ycsb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ycsb b/bin/ycsb index f5b5227384..99922122ee 100755 --- a/bin/ycsb +++ b/bin/ycsb @@ -237,7 +237,7 @@ def main(): p.add_argument('-cp', dest='classpath', help="""Additional classpath entries, e.g. '-cp /tmp/hbase-1.0.1.1/conf'. Will be prepended to the YCSB classpath.""") - p.add_argument("-jvm-args", default="-javaagent:/tmp/ycsb/ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/lib/applicationinsights-agent-3.5.1.jar", type=shlex.split, + p.add_argument("-jvm-args", default=[], type=shlex.split, help="""Additional arguments to pass to 'java', e.g. '-Xmx4g'""") p.add_argument("command", choices=sorted(COMMANDS), From 0e3e853f85a4e81ff277a2eb39ea2c9612ce116a Mon Sep 17 00:00:00 2001 From: Darshan Patnekar Date: Wed, 20 Mar 2024 11:34:40 -0700 Subject: [PATCH 68/72] added setenv.sh file for JAVA_OPTS --- bin/setenv.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 bin/setenv.sh diff --git a/bin/setenv.sh b/bin/setenv.sh new file mode 100644 index 0000000000..80d893ff19 --- /dev/null +++ b/bin/setenv.sh @@ -0,0 +1,3 @@ +export APPLICATIONINSIGHTS_CONNECTION_STRING="InstrumentationKey=cca9d630-bbfa-4f10-94c1-f602f57f1885;IngestionEndpoint=https://eastus2-3.in.applicationinsights.azure.com/;LiveEndpoint=https://eastus2.livediagnostics.monitor.azure.com/" +export APPLICATIONINSIGHTS_METRIC_INTERVAL_SECONDS=10 +export JAVA_OPTS=-javaagent:"/tmp/ycsb/ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/lib/applicationinsights-agent-3.5.1.jar" \ No newline at end of file From 7c824e5080aef85067486d5de51d4e973a58f40c Mon Sep 17 00:00:00 2001 From: Darshan Patnekar Date: Thu, 21 Mar 2024 00:42:44 -0700 Subject: [PATCH 69/72] updated pom file --- azurecosmos/pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azurecosmos/pom.xml b/azurecosmos/pom.xml index 155eea5cf6..5c8da86b33 100644 --- a/azurecosmos/pom.xml +++ b/azurecosmos/pom.xml @@ -58,8 +58,9 @@ LICENSE file. io.micrometer - micrometer-core - 1.6.1 + micrometer-registry-azure-monitor + 1.9.2 + compile com.microsoft.azure From 52ce2e30caaf820185792cefd4840883ba7c297c Mon Sep 17 00:00:00 2001 From: Darshan Patnekar Date: Thu, 21 Mar 2024 09:16:25 -0700 Subject: [PATCH 70/72] removed the setenv.sh --- bin/setenv.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 bin/setenv.sh diff --git a/bin/setenv.sh b/bin/setenv.sh deleted file mode 100644 index 80d893ff19..0000000000 --- a/bin/setenv.sh +++ /dev/null @@ -1,3 +0,0 @@ -export APPLICATIONINSIGHTS_CONNECTION_STRING="InstrumentationKey=cca9d630-bbfa-4f10-94c1-f602f57f1885;IngestionEndpoint=https://eastus2-3.in.applicationinsights.azure.com/;LiveEndpoint=https://eastus2.livediagnostics.monitor.azure.com/" -export APPLICATIONINSIGHTS_METRIC_INTERVAL_SECONDS=10 -export JAVA_OPTS=-javaagent:"/tmp/ycsb/ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/lib/applicationinsights-agent-3.5.1.jar" \ No newline at end of file From 95c9b416285c24a8b758826ef194fc162e13766b Mon Sep 17 00:00:00 2001 From: Darshan Patnekar Date: Thu, 21 Mar 2024 18:08:45 -0700 Subject: [PATCH 71/72] added details in readme for sending telemetry to appInsights --- azurecosmos/README.md | 20 +++++++++++++++++++- azurecosmos/conf/azurecosmos.properties | 6 +++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/azurecosmos/README.md b/azurecosmos/README.md index 9af889b4c5..1ff0575bcd 100644 --- a/azurecosmos/README.md +++ b/azurecosmos/README.md @@ -122,7 +122,25 @@ These parameters are also defined in a template configuration file in the following location: $YCSB_HOME/azurecosmos/conf/azurecosmos.properties -### 3. FAQs +### 3. Sending YCSB Telemetry to Azure Application Insights +- YCSB supports sending telemetry data like operation metrics & Cosmos client diagnostic logs to Azure Application Insights. +- In order to send the telemetry data to Azure Application Insights, you need to set the following parameters in the azurecosmos/conf/azurecosmos.properties file: + + | Parameter | Default Value | + |-------------------------------------------------|----------| + | azurecosmos.appInsightConnectionString | "" | + | azurecosmos.pointOperationLatencyThresholdInMS | 100 | + | azurecosmos.nonPointOperationLatencyThresholdInMS | 500 | + | azurecosmos.requestChargeThreshold | 100 | + +- Also, need to set the following environment variables on the machine where YCSB is running: +```bash +echo 'export APPLICATIONINSIGHTS_CONNECTION_STRING=""' >> ~/.profile +echo 'export APPLICATIONINSIGHTS_METRIC_INTERVAL_SECONDS=' >> ~/.profile +echo 'export JAVA_OPTS=-javaagent:"/tmp/ycsb/ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/lib/applicationinsights-agent-3.5.1.jar"' >> ~/.profile +source ~/.profile +``` +- **Please do not change the JAVA_OPTS value above as it is required for the telemetry to work.** ### 4. Example command ./bin/ycsb run azurecosmos -P workloads/workloadc -p azurecosmos.primaryKey= -p azurecosmos.uri=https://.documents.azure.com:443/ -p recordcount=100 -p operationcount=100 diff --git a/azurecosmos/conf/azurecosmos.properties b/azurecosmos/conf/azurecosmos.properties index 2b5e8f140c..f3a79a632f 100644 --- a/azurecosmos/conf/azurecosmos.properties +++ b/azurecosmos/conf/azurecosmos.properties @@ -94,9 +94,9 @@ # azurecosmos.appInsightConnectionString = # Cosmos Client Diagnostics logs options -# azurecosmos.pointOperationLatencyThresholdInMS = 100 -# azurecosmos.nonPointOperationLatencyThresholdInMS = 500 -# azurecosmos.requestChargeThreshold = 100 +# azurecosmos.pointOperationLatencyThresholdInMS = +# azurecosmos.nonPointOperationLatencyThresholdInMS = +# azurecosmos.requestChargeThreshold = # Sets the comma separated preferred region list. # Default value is null. From 91d52cd610c770b0ded17e6cbe9233a75dbbd1d8 Mon Sep 17 00:00:00 2001 From: Darshan Patnekar Date: Mon, 25 Mar 2024 14:48:53 -0700 Subject: [PATCH 72/72] Addressed PR comment --- azurecosmos/README.md | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/azurecosmos/README.md b/azurecosmos/README.md index 1ff0575bcd..25838f223e 100644 --- a/azurecosmos/README.md +++ b/azurecosmos/README.md @@ -122,25 +122,7 @@ These parameters are also defined in a template configuration file in the following location: $YCSB_HOME/azurecosmos/conf/azurecosmos.properties -### 3. Sending YCSB Telemetry to Azure Application Insights -- YCSB supports sending telemetry data like operation metrics & Cosmos client diagnostic logs to Azure Application Insights. -- In order to send the telemetry data to Azure Application Insights, you need to set the following parameters in the azurecosmos/conf/azurecosmos.properties file: - - | Parameter | Default Value | - |-------------------------------------------------|----------| - | azurecosmos.appInsightConnectionString | "" | - | azurecosmos.pointOperationLatencyThresholdInMS | 100 | - | azurecosmos.nonPointOperationLatencyThresholdInMS | 500 | - | azurecosmos.requestChargeThreshold | 100 | - -- Also, need to set the following environment variables on the machine where YCSB is running: -```bash -echo 'export APPLICATIONINSIGHTS_CONNECTION_STRING=""' >> ~/.profile -echo 'export APPLICATIONINSIGHTS_METRIC_INTERVAL_SECONDS=' >> ~/.profile -echo 'export JAVA_OPTS=-javaagent:"/tmp/ycsb/ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/lib/applicationinsights-agent-3.5.1.jar"' >> ~/.profile -source ~/.profile -``` -- **Please do not change the JAVA_OPTS value above as it is required for the telemetry to work.** +### 3. FAQs ### 4. Example command -./bin/ycsb run azurecosmos -P workloads/workloadc -p azurecosmos.primaryKey= -p azurecosmos.uri=https://.documents.azure.com:443/ -p recordcount=100 -p operationcount=100 +./bin/ycsb run azurecosmos -P workloads/workloadc -p azurecosmos.primaryKey= -p azurecosmos.uri=https://.documents.azure.com:443/ -p recordcount=100 -p operationcount=100 \ No newline at end of file