Skip to content

Commit 6f4c00a

Browse files
mahendra-googlemahendra-google
committed
add samples and tets for storage batch operations
1 parent 0cfed1b commit 6f4c00a

16 files changed

+1018
-0
lines changed

‎storagebatchoperations/api/README.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Cloud Storage Batch Operations Sample
2+
3+
These samples demonstrate how to interact with the [Google Cloud Storage Batch Operations API][Storage Batch Operations] from C# and
4+
the .NET client libraries to call the Storage Batch Operations API.
5+
6+
The samples requires [.NET 8][.NET].That means using
7+
[Visual Studio 2022](https://www.visualstudio.com/), or the command line.
8+
9+
## Setup
10+
11+
1. Set up a [.NET development environment](https://cloud.google.com/dotnet/docs/setup).
12+
13+
2. Enable APIs for your project.
14+
[Click here][enable-api]
15+
to visit Cloud Platform Console and enable the Google Cloud Storage Batch Operations API.
16+
17+
## Contributing changes
18+
19+
* See [CONTRIBUTING.md](../../CONTRIBUTING.md)
20+
21+
## Licensing
22+
23+
* See [LICENSE](../../LICENSE)
24+
25+
## Testing
26+
27+
* See [TESTING.md](../../TESTING.md)
28+
29+
[Storage Batch Operations]: https://cloud.google.com/storage/docs/batch-operations/overview
30+
[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid=storagebatchoperations_api&showconfirmation=true
31+
[.NET]: https://dotnet.microsoft.com/en-us/download

‎storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License").
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
using Google.Api.Gax.ResourceNames;
16+
using Google.Cloud.StorageBatchOperations.V1;
17+
using Google.LongRunning;
18+
using System.IO;
19+
using System.Text;
20+
using System.Threading;
21+
using Xunit;
22+
23+
[Collection(nameof(StorageFixture))]
24+
public class CancelBatchJobTest
25+
{
26+
private readonly StorageFixture _fixture;
27+
private readonly BucketList.Types.Bucket _bucket = new();
28+
private readonly BucketList _bucketList = new();
29+
private readonly PrefixList _prefixListObject = new();
30+
31+
public CancelBatchJobTest(StorageFixture fixture)
32+
{
33+
int i = 10;
34+
_fixture = fixture;
35+
var bucketName = _fixture.GenerateBucketName();
36+
_fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true);
37+
// Uploading objects to the bucket.
38+
while (i >= 0)
39+
{
40+
var objectName = _fixture.GenerateGuid();
41+
var objectContent = _fixture.GenerateGuid();
42+
byte[] byteObjectContent = Encoding.UTF8.GetBytes(objectContent);
43+
MemoryStream streamObjectContent = new MemoryStream(byteObjectContent);
44+
_fixture.Client.UploadObject(bucketName, objectName, "application/text", streamObjectContent);
45+
i--;
46+
}
47+
_bucket = new BucketList.Types.Bucket
48+
{
49+
Bucket_ = bucketName,
50+
// The prefix list is used to specify the objects to be deleted. To match all objects, use an empty list.
51+
PrefixList = _prefixListObject
52+
};
53+
// Adding the bucket to the bucket list.
54+
_bucketList.Buckets.Insert(0, _bucket);
55+
}
56+
57+
[Fact]
58+
public void TestCancelBatchJob()
59+
{
60+
CancelBatchJobSample cancelBatchJob = new CancelBatchJobSample();
61+
ListBatchJobsSample listBatchJobs = new ListBatchJobsSample();
62+
GetBatchJobSample getBatchJob = new GetBatchJobSample();
63+
64+
string filter = "state:canceled";
65+
int pageSize = 10;
66+
string orderBy = "create_time";
67+
68+
var jobId = _fixture.GenerateGuid();
69+
var createdJob = CreateBatchJob(_fixture.LocationName, _bucketList, jobId);
70+
var cancelJobResponse = cancelBatchJob.CancelBatchJob(createdJob);
71+
PollUntilCancelled();
72+
var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy);
73+
Assert.Contains(batchJobs, job => job.Name == createdJob && job.State == Job.Types.State.Canceled);
74+
Job cancelledJob = getBatchJob.GetBatchJob(createdJob);
75+
Assert.Equal(createdJob, cancelledJob.Name.ToString());
76+
Assert.Equal("Canceled", cancelledJob.State.ToString());
77+
_fixture.DeleteBatchJob(createdJob);
78+
}
79+
80+
/// <summary>
81+
/// Create a batch job with the specified transformation case and bucket list.
82+
/// </summary>
83+
public static string CreateBatchJob(LocationName locationName,
84+
BucketList bucketList,
85+
string jobId = "12345678910")
86+
{
87+
StorageBatchOperationsClient storageBatchClient = StorageBatchOperationsClient.Create();
88+
// Creates a batch job with the specified bucket list and delete object settings.
89+
CreateJobRequest request = new CreateJobRequest
90+
{
91+
ParentAsLocationName = locationName,
92+
JobId = jobId,
93+
Job = new Job
94+
{
95+
BucketList = bucketList,
96+
DeleteObject = new DeleteObject { PermanentObjectDeletionEnabled = true }
97+
},
98+
RequestId = jobId,
99+
};
100+
101+
Operation<Job, OperationMetadata> response = storageBatchClient.CreateJob(request);
102+
string operationName = response.Name;
103+
Operation<Job, OperationMetadata> retrievedResponse = storageBatchClient.PollOnceCreateJob(operationName);
104+
// Poll until we get storage batch operation job name.
105+
while (true)
106+
{
107+
retrievedResponse = retrievedResponse.PollOnce();
108+
if (string.IsNullOrEmpty(retrievedResponse.Metadata.Job.Name.ToString()))
109+
{
110+
continue;
111+
}
112+
else
113+
{
114+
break;
115+
}
116+
}
117+
118+
string jobName = retrievedResponse.Metadata.Job.Name;
119+
return jobName;
120+
}
121+
122+
/// <summary>
123+
/// Wait for 15 seconds until completion of cancel batch job operation.
124+
/// </summary>
125+
private static void PollUntilCancelled() => Thread.Sleep(15000);
126+
}

‎storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License").
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
using Google.Cloud.StorageBatchOperations.V1;
16+
using System;
17+
using System.IO;
18+
using System.Text;
19+
using Xunit;
20+
21+
[Collection(nameof(StorageFixture))]
22+
public class CreateBatchJobTest
23+
{
24+
private readonly StorageFixture _fixture;
25+
private readonly BucketList.Types.Bucket _bucket = new();
26+
private readonly BucketList _bucketList = new();
27+
private readonly PrefixList _prefixListObject = new();
28+
private string _kmsKey;
29+
private string _keyRingId;
30+
private string _cryptoKeyId;
31+
private CryptoKeyName _cryptoKeyName;
32+
33+
public CreateBatchJobTest(StorageFixture fixture)
34+
{
35+
_fixture = fixture;
36+
var bucketName = _fixture.GenerateBucketName();
37+
_fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true);
38+
39+
var manifestBucketName = _fixture.GenerateBucketName();
40+
_fixture.CreateBucket(manifestBucketName, multiVersion: false, softDelete: false, registerForDeletion: true);
41+
42+
var objectName = _fixture.GenerateGuid();
43+
var manifestObjectName = _fixture.GenerateGuid();
44+
var objectContent = _fixture.GenerateGuid();
45+
var manifestObjectContent = $"bucket,name,generation{Environment.NewLine}{bucketName},{objectName}";
46+
47+
byte[] byteObjectContent = Encoding.UTF8.GetBytes(objectContent);
48+
MemoryStream streamObjectContent = new MemoryStream(byteObjectContent);
49+
// Uploading an object to the bucket
50+
_fixture.Client.UploadObject(bucketName, objectName, "application/text", streamObjectContent);
51+
52+
byte[] byteManifestObjectContent = Encoding.UTF8.GetBytes(manifestObjectContent);
53+
// Uploading a manifest object to the manifest bucket
54+
MemoryStream streamManifestObjectContent = new MemoryStream(byteManifestObjectContent);
55+
_fixture.Client.UploadObject(manifestBucketName, $"{manifestObjectName}.csv", "text/csv", streamManifestObjectContent);
56+
_bucket = new BucketList.Types.Bucket
57+
{
58+
Bucket_ = bucketName,
59+
// The prefix list is used to specify the objects to be transformed. To match all objects, use an empty list.
60+
PrefixList = _prefixListObject,
61+
// Manifest location contains csv file having list of objects to be transformed"
62+
Manifest = new Manifest { ManifestLocation = $"gs://{manifestBucketName}/{manifestObjectName}.csv" }
63+
};
64+
// Adding the bucket to the bucket list.
65+
_bucketList.Buckets.Insert(0, _bucket);
66+
}
67+
68+
[Fact]
69+
public void TestCreateBatchJob()
70+
{
71+
CreateBatchJobSample createJob = new CreateBatchJobSample();
72+
var jobId = _fixture.GenerateGuid();
73+
var jobTransformationCase = "DeleteObject";
74+
var holdState = "EventBasedHoldSet";
75+
var jobTransformationObject = new object();
76+
string jobType;
77+
78+
// If the job transformation case is PutObjectHold, we can set the hold state to EventBasedHoldSet or EventBasedHoldUnSet or TemporaryHoldSet or TemporaryHoldUnSet.
79+
if (jobTransformationCase == "PutObjectHold")
80+
{
81+
jobType = $"{jobTransformationCase}{holdState}";
82+
}
83+
// If the job transformation case is other than PutObjectHold, we dont set the hold state.
84+
else
85+
{
86+
jobType = jobTransformationCase;
87+
}
88+
// If the job transformation case is RewriteObject, we can set the KmsKey and KmsKeyAsCryptoKeyName.
89+
if (jobTransformationCase == "RewriteObject")
90+
{
91+
_keyRingId = GetEnvironmentVariable("STORAGE_KMS_KEYRING_ID", "This is the Key Ring ID");
92+
_cryptoKeyId = GetEnvironmentVariable("STORAGE_KMS_CRYPTOKEY_ID", "This is the Crypto Key ID.");
93+
_kmsKey = $"projects/{_fixture.ProjectId}/locations/{_fixture.LocationId}/keyRings/{_keyRingId}/cryptoKeys/{_cryptoKeyId}";
94+
_cryptoKeyName = CryptoKeyName.FromProjectLocationKeyRingCryptoKey(_fixture.ProjectId, _fixture.LocationId, _keyRingId, _cryptoKeyId);
95+
RewriteObject rewriteObject = new RewriteObject { KmsKey = _kmsKey, KmsKeyAsCryptoKeyName = _cryptoKeyName };
96+
jobTransformationObject = rewriteObject;
97+
98+
}
99+
// If the job transformation case is PutMetadata, we can set the CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType and CustomTime.
100+
else if (jobTransformationCase == "PutMetadata")
101+
{
102+
PutMetadata putMetadata = new PutMetadata
103+
{
104+
CacheControl = "no-cache",
105+
ContentDisposition = "inline",
106+
ContentEncoding = "gzip",
107+
ContentLanguage = "en-US",
108+
ContentType = "text/plain",
109+
CustomTime = DateTime.UtcNow.ToString("o")
110+
};
111+
jobTransformationObject = putMetadata;
112+
}
113+
// Create a batch job with the specified transformation case and bucket list
114+
var createdBatchJob = createJob.CreateBatchJob(_fixture.LocationName, _bucketList, jobId, jobType, jobTransformationObject);
115+
Assert.Equal(createdBatchJob.BucketList, _bucketList);
116+
Assert.Equal(createdBatchJob.TransformationCase.ToString(), jobTransformationCase);
117+
Assert.Equal(createdBatchJob.SourceCase.ToString(), _bucketList.GetType().Name);
118+
Assert.NotNull(createdBatchJob.Name);
119+
Assert.NotNull(createdBatchJob.JobName);
120+
Assert.NotNull(createdBatchJob.CreateTime);
121+
Assert.NotNull(createdBatchJob.CompleteTime);
122+
_fixture.DeleteBatchJob(createdBatchJob.Name);
123+
}
124+
125+
private static string GetEnvironmentVariable(string envVarName, string message = "")
126+
{
127+
string varValue = Environment.GetEnvironmentVariable(envVarName);
128+
if (string.IsNullOrEmpty(varValue))
129+
{
130+
throw new InvalidOperationException(
131+
$"Please set the {envVarName} environment variable. {message}");
132+
}
133+
return varValue;
134+
}
135+
}

‎storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License").
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
using Google.Cloud.StorageBatchOperations.V1;
16+
using Xunit;
17+
18+
[Collection(nameof(StorageFixture))]
19+
public class DeleteBatchJobTest
20+
{
21+
private readonly StorageFixture _fixture;
22+
private readonly BucketList.Types.Bucket _bucket = new();
23+
private readonly BucketList _bucketList = new();
24+
private readonly PrefixList _prefixListObject = new();
25+
26+
public DeleteBatchJobTest(StorageFixture fixture)
27+
{
28+
_fixture = fixture;
29+
var bucketName = _fixture.GenerateBucketName();
30+
_fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true);
31+
_bucket = new BucketList.Types.Bucket
32+
{
33+
Bucket_ = bucketName,
34+
// The prefix list is used to specify the objects to be deleted. To match all objects, use an empty list.
35+
PrefixList = _prefixListObject
36+
};
37+
// Adding the bucket to the bucket list.
38+
_bucketList.Buckets.Insert(0, _bucket);
39+
}
40+
41+
[Fact]
42+
public void TestDeleteBatchJob()
43+
{
44+
DeleteBatchJobSample deleteBatchJob = new DeleteBatchJobSample();
45+
ListBatchJobsSample listBatchJobs = new ListBatchJobsSample();
46+
GetBatchJobSample getBatchJob = new GetBatchJobSample();
47+
CreateBatchJobSample createBatchJob = new CreateBatchJobSample();
48+
49+
string filter = "";
50+
int pageSize = 10;
51+
string orderBy = "create_time";
52+
53+
var jobId = _fixture.GenerateGuid();
54+
var createdJob = createBatchJob.CreateBatchJob(_fixture.LocationName, _bucketList, jobId);
55+
// Delete the created job.
56+
deleteBatchJob.DeleteBatchJob(createdJob.Name);
57+
var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy);
58+
// Verify that the job is deleted.
59+
Assert.DoesNotContain(batchJobs, job => job.JobName == createdJob.JobName);
60+
// Attempt to get the deleted job, which should throw an exception.
61+
var exception = Assert.Throws<Grpc.Core.RpcException>(() => getBatchJob.GetBatchJob(createdJob.Name));
62+
Assert.Equal(Grpc.Core.StatusCode.NotFound, exception.StatusCode);
63+
}
64+
}

0 commit comments

Comments
 (0)