Skip to content

Commit c438ac2

Browse files
jockechjockech
authored
add e2e test for amazon-s3-plugins (#136)
* add e2e test for amazon-s3-plugins * fix comments * fix comments * close stream
1 parent a6fee97 commit c438ac2

File tree

15 files changed

+486
-1
lines changed

15 files changed

+486
-1
lines changed

‎pom.xml‎

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,12 @@
8585
</properties>
8686

8787
<dependencies>
88+
<dependency>
89+
<groupId>com.google.guava</groupId>
90+
<artifactId>guava</artifactId>
91+
<version>27.0.1-jre</version>
92+
<scope>test</scope>
93+
</dependency>
8894
<dependency>
8995
<groupId>io.cdap.cdap</groupId>
9096
<artifactId>cdap-etl-api</artifactId>
@@ -238,6 +244,7 @@
238244
</dependencies>
239245

240246
<build>
247+
<testSourceDirectory>${testSourceLocation}</testSourceDirectory>
241248
<pluginManagement>
242249
<plugins>
243250
<plugin>
@@ -252,7 +259,7 @@
252259
<plugin>
253260
<groupId>org.apache.felix</groupId>
254261
<artifactId>maven-bundle-plugin</artifactId>
255-
<version>2.5.4</version>
262+
<version>3.5.0</version>
256263
<extensions>true</extensions>
257264
<configuration>
258265
<instructions>
@@ -357,5 +364,96 @@
357364
</plugins>
358365
</build>
359366
</profile>
367+
<profile>
368+
<id>e2e-tests</id>
369+
<properties>
370+
<testSourceLocation>src/e2e-test/java</testSourceLocation>
371+
</properties>
372+
<build>
373+
<testResources>
374+
<testResource>
375+
<directory>src/e2e-test/resources</directory>
376+
</testResource>
377+
</testResources>
378+
<plugins>
379+
<plugin>
380+
<groupId>org.apache.maven.plugins</groupId>
381+
<artifactId>maven-surefire-plugin</artifactId>
382+
<version>2.18.1</version>
383+
<configuration>
384+
<skipTests>true</skipTests>
385+
</configuration>
386+
</plugin>
387+
388+
<plugin>
389+
<groupId>org.apache.maven.plugins</groupId>
390+
<artifactId>maven-failsafe-plugin</artifactId>
391+
<version>3.0.0-M5</version>
392+
<configuration>
393+
<includes>
394+
<include>TestRunner.java</include>
395+
</includes>
396+
<!--Start configuration to run TestRunners in parallel-->
397+
<parallel>classes</parallel> <!--Running TestRunner classes in parallel-->
398+
<threadCount>2</threadCount> <!--Number of classes to run in parallel-->
399+
<forkCount>2</forkCount> <!--Number of JVM processes -->
400+
<reuseForks>true</reuseForks>
401+
<!--End configuration to run TestRunners in parallel-->
402+
</configuration>
403+
<executions>
404+
<execution>
405+
<goals>
406+
<goal>integration-test</goal>
407+
</goals>
408+
</execution>
409+
</executions>
410+
</plugin>
411+
412+
<plugin>
413+
<groupId>net.masterthought</groupId>
414+
<artifactId>maven-cucumber-reporting</artifactId>
415+
<version>5.5.0</version>
416+
417+
<executions>
418+
<execution>
419+
<id>execution</id>
420+
<phase>verify</phase>
421+
<goals>
422+
<goal>generate</goal>
423+
</goals>
424+
<configuration>
425+
<projectName>Cucumber Reports</projectName> <!-- Replace with project name -->
426+
<outputDirectory>target/cucumber-reports/advanced-reports</outputDirectory>
427+
<buildNumber>1</buildNumber>
428+
<skip>false</skip>
429+
<inputDirectory>${project.build.directory}/cucumber-reports</inputDirectory>
430+
<jsonFiles> <!-- supports wildcard or name pattern -->
431+
<param>**/*.json</param>
432+
</jsonFiles> <!-- optional, defaults to outputDirectory if not specified -->
433+
<classificationDirectory>${project.build.directory}/cucumber-reports</classificationDirectory>
434+
<checkBuildResult>true</checkBuildResult>
435+
</configuration>
436+
</execution>
437+
</executions>
438+
</plugin>
439+
</plugins>
440+
</build>
441+
442+
<dependencies>
443+
<dependency>
444+
<groupId>io.cdap.tests.e2e</groupId>
445+
<artifactId>cdap-e2e-framework</artifactId>
446+
<version>0.0.1-SNAPSHOT</version>
447+
<scope>test</scope>
448+
</dependency>
449+
<dependency>
450+
<groupId>ch.qos.logback</groupId>
451+
<artifactId>logback-classic</artifactId>
452+
<version>1.2.8</version>
453+
<scope>runtime</scope>
454+
</dependency>
455+
</dependencies>
456+
457+
</profile>
360458
</profiles>
361459
</project>

‎src/e2e-test/features/S3ToS3.feature‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
@S3
2+
Feature: S3 source - Verification of S3 to S3 successful data transfer
3+
@S3_SOURCE_TEST @S3_SINK_TEST
4+
Scenario:Validate successful records transfer from S3 to S3
5+
Given Open Datafusion Project to configure pipeline
6+
When Select plugin: "Amazon S3" from the plugins list as: "Source"
7+
When Expand Plugin group in the LHS plugins list: "Sink"
8+
When Select plugin: "Amazon S3" from the plugins list as: "Sink"
9+
Then Connect source as "Amazon-S3" and sink as "S3" to establish connection
10+
Then Open Amazon S3 source properties
11+
Then Enter input plugin property: "accessID" with value: "accessID"
12+
Then Enter input plugin property: "accessKey" with value: "accessKey"
13+
Then Enter input plugin property: "referenceName" with value: "s3Source"
14+
Then Enter input plugin property: "path" with value: "sourcePath"
15+
Then Select dropdown plugin property: "format" with option value: "csv"
16+
Then Click plugin property: "skipHeader"
17+
Then Click plugin property: "enableQuotedValues"
18+
Then Click on the Get Schema button
19+
Then Verify the Output Schema matches the Expected Schema: "outputSchema"
20+
Then Validate "S3" plugin properties
21+
Then Close the Plugin Properties page
22+
Then Open Amazon S3 sink properties
23+
Then Enter input plugin property: "accessID" with value: "accessID"
24+
Then Enter input plugin property: "accessKey" with value: "accessKey"
25+
Then Enter input plugin property: "referenceName" with value: "s3Sink"
26+
Then Enter input plugin property: "path" with value: "sinkPath"
27+
Then Select dropdown plugin property: "format" with option value: "csv"
28+
Then Validate "S3" plugin properties
29+
Then Close the Plugin Properties page
30+
Then Save the pipeline
31+
Then Preview and run the pipeline
32+
Then Verify the preview of pipeline is "success"
33+
Then Click on preview data for S3 sink
34+
Then Verify preview output schema matches the outputSchema captured in properties
35+
Then Close the preview data
36+
Then Deploy the pipeline
37+
Then Run the Pipeline in Runtime
38+
Then Wait till pipeline is in running state
39+
Then Open and capture logs
40+
Then Verify the pipeline status is "Succeeded"
41+
Then Verify csv data is transferred to target S3 bucket

‎src/e2e-test/java/io/cdap/plugin/S3Client.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package io.cdap.plugin;
2+
3+
import com.amazonaws.auth.AWSCredentials;
4+
import com.amazonaws.auth.AWSStaticCredentialsProvider;
5+
import com.amazonaws.auth.BasicAWSCredentials;
6+
import com.amazonaws.services.s3.AmazonS3;
7+
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
8+
import com.amazonaws.services.s3.model.S3ObjectInputStream;
9+
import com.amazonaws.services.s3.model.S3ObjectSummary;
10+
import io.cdap.e2e.utils.PluginPropertyUtils;
11+
import io.cdap.plugin.aws.s3.common.S3ConnectorConfig;
12+
13+
import java.io.File;
14+
import java.io.IOException;
15+
import java.net.URISyntaxException;
16+
import java.nio.file.Paths;
17+
import java.util.List;
18+
import java.util.Objects;
19+
20+
/**
21+
* S3 client for e2e test.
22+
*/
23+
public class S3Client {
24+
private static AmazonS3 s3Client = null;
25+
26+
public S3Client() {
27+
}
28+
29+
private static S3ConnectorConfig initializeConfig() {
30+
return new S3ConnectorConfig(PluginPropertyUtils.pluginProp("accessID"),
31+
PluginPropertyUtils.pluginProp("accessKey"), null, "Access Credentials",
32+
PluginPropertyUtils.pluginProp("region"));
33+
}
34+
35+
private static AmazonS3 getS3Client() throws IOException {
36+
if (s3Client == null) {
37+
AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
38+
S3ConnectorConfig config = initializeConfig();
39+
assert config.getRegion() != null;
40+
builder.setRegion(config.getRegion());
41+
AWSCredentials creds;
42+
assert config.getAccessID() != null;
43+
assert config.getAccessKey() != null;
44+
creds = new BasicAWSCredentials(config.getAccessID(), config.getAccessKey());
45+
s3Client = builder.withCredentials(new AWSStaticCredentialsProvider(creds)).build();
46+
}
47+
return s3Client;
48+
}
49+
50+
public static String createBucket(String bucketName) throws IOException {
51+
return getS3Client().createBucket(bucketName).getName();
52+
}
53+
54+
public static void deleteBucket(String bucketName) throws IOException {
55+
AmazonS3 s3Client = getS3Client();
56+
for (S3ObjectSummary s3ObjectSummary: s3Client.listObjects(bucketName).getObjectSummaries()) {
57+
s3Client.deleteObject(bucketName, s3ObjectSummary.getKey());
58+
}
59+
s3Client.deleteBucket(bucketName);
60+
}
61+
62+
63+
public static void uploadObject(String bucketName, String filePath) throws URISyntaxException, IOException {
64+
File file = new File(
65+
String.valueOf(Paths.get(Objects.requireNonNull(S3Client.class.getResource("/" + filePath)).toURI()))
66+
);
67+
AmazonS3 s3Client = getS3Client();
68+
s3Client.putObject(bucketName, filePath, file);
69+
}
70+
71+
public static List<S3ObjectSummary> listObjects(String s3Bucket) throws IOException {
72+
return getS3Client().listObjects(s3Bucket).getObjectSummaries();
73+
}
74+
75+
public static S3ObjectInputStream getObjectContent(String bucketName, String key) throws IOException {
76+
return getS3Client().getObject(bucketName, key).getObjectContent();
77+
}
78+
}

‎src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package io.cdap.plugin.common.stepsdesign;
2+
3+
import io.cdap.e2e.utils.PluginPropertyUtils;
4+
import io.cdap.plugin.S3Client;
5+
import io.cucumber.java.After;
6+
import io.cucumber.java.Before;
7+
import org.apache.commons.lang.StringUtils;
8+
import stepsdesign.BeforeActions;
9+
10+
import java.io.IOException;
11+
import java.net.URISyntaxException;
12+
import java.util.UUID;
13+
14+
/**
15+
* S3 test hooks.
16+
*/
17+
public class TestSetupHooks {
18+
public static String s3SourceBucketName = StringUtils.EMPTY;
19+
public static String s3TargetBucketName = StringUtils.EMPTY;
20+
21+
@Before(order = 0)
22+
public static void overrideAWSCredentialsIfProvided() {
23+
String accessKey = System.getenv("accessKey");
24+
if (accessKey != null && !accessKey.isEmpty()) {
25+
PluginPropertyUtils.addPluginProp("accessKey", accessKey);
26+
}
27+
String accessID = System.getenv("accessID");
28+
if (accessID != null && !accessID.isEmpty()) {
29+
PluginPropertyUtils.addPluginProp("accessID", accessID);
30+
}
31+
}
32+
33+
@Before(order = 1, value = "@S3_SOURCE_TEST")
34+
public static void createS3BucketWithCSVFile() throws URISyntaxException, IOException {
35+
s3SourceBucketName = S3Client.createBucket("e2e-test-" + UUID.randomUUID());
36+
String filePath = PluginPropertyUtils.pluginProp("testCSV");
37+
S3Client.uploadObject(s3SourceBucketName, filePath);
38+
BeforeActions.scenario.write(
39+
"Created S3 Source Bucket " + s3SourceBucketName + " containing " + filePath + " file"
40+
);
41+
PluginPropertyUtils.addPluginProp(
42+
"sourcePath", "s3a://" + s3SourceBucketName + "/" + PluginPropertyUtils.pluginProp("sourcePath")
43+
);
44+
}
45+
46+
@Before(order = 1, value = "@S3_SINK_TEST")
47+
public static void createTempTargetS3Bucket() throws IOException {
48+
s3TargetBucketName = S3Client.createBucket("e2e-test-" + UUID.randomUUID());
49+
BeforeActions.scenario.write("Created S3 Target Bucket " + s3TargetBucketName);
50+
PluginPropertyUtils.addPluginProp("sinkPath", "s3a://" + s3TargetBucketName);
51+
}
52+
53+
54+
@After(order = 1, value = "@S3_SOURCE_TEST")
55+
public static void deleteS3BucketWithCSVFile() throws IOException {
56+
S3Client.deleteBucket(s3SourceBucketName);
57+
BeforeActions.scenario.write("Deleted S3 Source Bucket " + s3SourceBucketName);
58+
s3SourceBucketName = StringUtils.EMPTY;
59+
60+
}
61+
62+
@After(order = 1, value = "@S3_SINK_TEST")
63+
public static void deleteTempTargetS3Bucket() throws IOException {
64+
S3Client.deleteBucket(s3TargetBucketName);
65+
BeforeActions.scenario.write("Deleted S3 Target Bucket " + s3TargetBucketName);
66+
s3TargetBucketName = StringUtils.EMPTY;
67+
}
68+
}

‎src/e2e-test/java/io/cdap/plugin/common/stepsdesign/package-info.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
/**
2+
* Package contains the stepDesign for the common features.
3+
*/
4+
package io.cdap.plugin.common.stepsdesign;

‎src/e2e-test/java/io/cdap/plugin/package-info.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright © 2022 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
/**
18+
* Package contains Amazon S3 client.
19+
*/
20+
package io.cdap.plugin;

‎src/e2e-test/java/io/cdap/plugin/s3/runners/TestRunner.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* Copyright © 2021 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.s3.runners;
17+
18+
import io.cucumber.junit.Cucumber;
19+
import io.cucumber.junit.CucumberOptions;
20+
import org.junit.runner.RunWith;
21+
22+
/**
23+
* Test Runner to execute S3 cases.
24+
*/
25+
@RunWith(Cucumber.class)
26+
@CucumberOptions(
27+
features = {"src/e2e-test/features"},
28+
glue = {"io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.s3.stepsdesign", "stepsdesign"},
29+
tags = {"@S3"},
30+
monochrome = true,
31+
plugin = {"pretty", "html:target/cucumber-html-report/s3",
32+
"json:target/cucumber-reports/cucumber-s3.json",
33+
"junit:target/cucumber-reports/cucumber-s3.xml"}
34+
)
35+
public class TestRunner {
36+
}

‎src/e2e-test/java/io/cdap/plugin/s3/runners/package-info.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
/**
2+
* Package contains the runner for the S3 features.
3+
*/
4+
package io.cdap.plugin.s3.runners;

0 commit comments

Comments
 (0)