Skip to content

Commit ce634a1

Browse files
vaikasvaikas
committed
WIP: Add scaffolding to stand up Trillian on k8s.
Signed-off-by: Ville Aikas <[email protected]>
1 parent 5e32f6f commit ce634a1

File tree

18 files changed

+869
-21
lines changed

18 files changed

+869
-21
lines changed

‎.github/workflows/kind-scaffolding-e2e-createtree.yml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: Kind scaffolding test
2+
3+
on:
4+
pull_request:
5+
branches: [ 'main', 'release-*' ]
6+
7+
permissions: read-all
8+
9+
jobs:
10+
e2e-tests:
11+
name: test tree creation with scaffolding
12+
runs-on: ubuntu-latest
13+
14+
strategy:
15+
fail-fast: false # Keep running if one leg fails.
16+
matrix:
17+
k8s-version:
18+
- v1.22.x
19+
- v1.23.x
20+
21+
env:
22+
REGISTRY_NAME: registry.local
23+
REGISTRY_PORT: 5000
24+
KO_DOCKER_REPO: registry.local:5000/trillian
25+
26+
steps:
27+
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2.4.0
28+
- uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923 # v2.2.0
29+
with:
30+
go-version: '1.17'
31+
check-latest: true
32+
33+
- uses: imjasonh/setup-ko@2c3450ca27f6e6f2b02e72a40f2163c281a1f675 # v0.4
34+
35+
- name: Install yq
36+
uses: mikefarah/yq@70403375d7b96075bd68b40c434807cff1593568 # v4.25.1
37+
38+
- name: Setup mirror
39+
uses: chainguard-dev/actions/setup-mirror@main
40+
with:
41+
mirror: mirror.gcr.io
42+
43+
- name: Setup kind cluster
44+
uses: chainguard-dev/actions/setup-kind@main
45+
with:
46+
k8s-version: ${{ matrix.k8s-version }}
47+
cluster-suffix: c${{ github.run_id }}.local
48+
49+
- name: Setup knative
50+
uses: chainguard-dev/actions/setup-knative@main
51+
with:
52+
k8s-version: ${{ matrix.k8s-version }}
53+
cluster-suffix: c${{ github.run_id }}.local
54+
55+
- name: Install Trillian
56+
run: |
57+
echo '::group:: install Trillian scaffolding'
58+
ko apply -BRf ./examples/deployment/kubernetes/scaffolding
59+
echo '::endgroup::'
60+
echo '::group:::' waiting for services to come up
61+
kubectl wait -n trillian-system --for=condition=Ready --timeout=5m ksvc --all
62+
echo '::endgroup::'
63+
64+
- name: Create a tree on it
65+
run: |
66+
echo '::group:: install create tree job'
67+
ko apply -BRf ./examples/deployment/kubernetes/createtree
68+
echo '::endgroup::'
69+
echo '::group:::' waiting for job to complete'
70+
kubectl wait -n createtree --for=condition=Complete --timeout=5m jobs createtree
71+
echo '::endgroup::'
72+
kubectl -n createtree get cm trillian-tree -ojsonpath='{.data.treeID}'
73+
74+
- name: Collect diagnostics
75+
if: ${{ failure() }}
76+
uses: chainguard-dev/actions/kind-diag@84c993eaf02da1c325854fb272a4df9184bd80fc # main

‎cmd/createtree-k8s/main.go‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
// Copyright 2022 The Sigstore Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package main
16+
17+
import (
18+
"context"
19+
"flag"
20+
"fmt"
21+
"time"
22+
23+
"github.com/google/trillian"
24+
"github.com/google/trillian/client"
25+
"github.com/google/trillian/client/rpcflags"
26+
"github.com/pkg/errors"
27+
"google.golang.org/grpc"
28+
"google.golang.org/protobuf/types/known/durationpb"
29+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30+
"k8s.io/client-go/kubernetes"
31+
"k8s.io/client-go/rest"
32+
"knative.dev/pkg/logging"
33+
"knative.dev/pkg/signals"
34+
"sigs.k8s.io/release-utils/version"
35+
)
36+
37+
const (
38+
// Key in the configmap holding the value of the tree.
39+
treeKey = "treeID"
40+
)
41+
42+
var (
43+
ns = flag.String("namespace", "rekor-system", "Namespace where to update the configmap in")
44+
cmname = flag.String("configmap", "rekor-config", "Name of the configmap where the treeID lives")
45+
adminServerAddr = flag.String("admin_server", "log-server.trillian-system.svc:80", "Address of the gRPC Trillian Admin Server (host:port)")
46+
treeState = flag.String("tree_state", trillian.TreeState_ACTIVE.String(), "State of the new tree")
47+
treeType = flag.String("tree_type", trillian.TreeType_LOG.String(), "Type of the new tree")
48+
displayName = flag.String("display_name", "", "Display name of the new tree")
49+
description = flag.String("description", "", "Description of the new tree")
50+
maxRootDuration = flag.Duration("max_root_duration", time.Hour, "Interval after which a new signed root is produced despite no submissions; zero means never")
51+
force = flag.Bool("force", false, "Force create a new tree and update configmap")
52+
)
53+
54+
func main() {
55+
flag.Parse()
56+
ctx := signals.NewContext()
57+
versionInfo := version.GetVersionInfo()
58+
logging.FromContext(ctx).Infof("running create_tree Version: %s GitCommit: %s BuildDate: %s", versionInfo.GitVersion, versionInfo.GitCommit, versionInfo.BuildDate)
59+
60+
config, err := rest.InClusterConfig()
61+
if err != nil {
62+
logging.FromContext(ctx).Fatalf("Failed to get InClusterConfig: %v", err)
63+
}
64+
clientset, err := kubernetes.NewForConfig(config)
65+
if err != nil {
66+
logging.FromContext(ctx).Fatalf("Failed to get clientset: %v", err)
67+
}
68+
cm, err := clientset.CoreV1().ConfigMaps(*ns).Get(ctx, *cmname, metav1.GetOptions{})
69+
if err != nil {
70+
logging.FromContext(ctx).Fatalf("Failed to get the configmap %s/%s: %v", *ns, *cmname, err)
71+
}
72+
73+
if cm.Data == nil {
74+
cm.Data = make(map[string]string)
75+
}
76+
if treeID, ok := cm.Data[treeKey]; ok && !*force {
77+
logging.FromContext(ctx).Infof("Found existing TreeID: %s", treeID)
78+
return
79+
}
80+
81+
tree, err := createTree(ctx)
82+
if err != nil {
83+
logging.FromContext(ctx).Fatalf("Failed to create the trillian tree: %v", err)
84+
}
85+
cm.Data[treeKey] = fmt.Sprint(tree.TreeId)
86+
logging.FromContext(ctx).Infof("Created a new tree %d updating configmap %s/%s", tree.TreeId, *ns, *cmname)
87+
88+
_, err = clientset.CoreV1().ConfigMaps(*ns).Update(ctx, cm, metav1.UpdateOptions{})
89+
if err != nil {
90+
logging.FromContext(ctx).Fatalf("Failed to update the configmap: %v", err)
91+
}
92+
}
93+
94+
func createTree(ctx context.Context) (*trillian.Tree, error) {
95+
req, err := newRequest(ctx)
96+
if err != nil {
97+
return nil, err
98+
}
99+
100+
dialOpts, err := rpcflags.NewClientDialOptionsFromFlags()
101+
if err != nil {
102+
return nil, errors.Wrap(err, "failed to determine dial options")
103+
}
104+
105+
conn, err := grpc.Dial(*adminServerAddr, dialOpts...)
106+
if err != nil {
107+
return nil, errors.Wrap(err, "failed to dial")
108+
}
109+
defer conn.Close()
110+
111+
adminClient := trillian.NewTrillianAdminClient(conn)
112+
logClient := trillian.NewTrillianLogClient(conn)
113+
114+
return client.CreateAndInitTree(ctx, req, adminClient, logClient)
115+
}
116+
117+
func newRequest(ctx context.Context) (*trillian.CreateTreeRequest, error) {
118+
ts, ok := trillian.TreeState_value[*treeState]
119+
if !ok {
120+
return nil, fmt.Errorf("unknown TreeState: %v", *treeState)
121+
}
122+
123+
tt, ok := trillian.TreeType_value[*treeType]
124+
if !ok {
125+
return nil, fmt.Errorf("unknown TreeType: %v", *treeType)
126+
}
127+
128+
ctr := &trillian.CreateTreeRequest{Tree: &trillian.Tree{
129+
TreeState: trillian.TreeState(ts),
130+
TreeType: trillian.TreeType(tt),
131+
DisplayName: *displayName,
132+
Description: *description,
133+
MaxRootDuration: durationpb.New(*maxRootDuration),
134+
}}
135+
logging.FromContext(ctx).Infof("Creating Tree: %+v", ctr.Tree)
136+
137+
return ctr, nil
138+
}

‎examples/deployment/kubernetes/README-SCAFFOLDING.md‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Deploying onto Kubernetes
2+
3+
This document guides you through the process of spinning up an example Trillian
4+
deployment on Kubernetes cluster with [Knative](https://knative.dev/docs/)
5+
installed. It's suitable for GitHub action based e2e tests
6+
as well as local testing using something like kind cluster.
7+
8+
## Prerequisites
9+
10+
1. You should have this repo checked out :)
11+
1. A Kubernetes cluster with [Knative](https://knative.dev/docs/) installed.
12+
1. [ko](https://github.com/google/ko) installed.
13+
1. You have `kubectl` installed.
14+
15+
## Process
16+
17+
1. Create the scaffolding parts of the Trillian system.
18+
```shell
19+
ko apply -BRf ./examples/deployment/kubernetes/scaffolding
20+
```
21+
This spins up a namespace `trillian-system` where it deploys
22+
the following components:
23+
* log-signer
24+
* log-server
25+
* mysql server
26+
2. Let's make sure everything comes up ready:
27+
```shell
28+
kubectl wait -n trillian-system --for=condition=Ready --timeout=5m ksvc --all
29+
```
30+
31+
You should see something like this:
32+
```shell
33+
vaikas@villes-mbp scaffolding % kubectl wait -n trillian-system --for=condition=Ready --timeout=5m ksvc --all
34+
service.serving.knative.dev/log-server condition met
35+
service.serving.knative.dev/log-signer condition met
36+
```
37+
3. Then create a tree in the Trillian:
38+
```shell
39+
ko apply -BRf ./examples/deployment/kubernetes/createtree
40+
```
41+
42+
And make sure it completes.
43+
44+
```shell
45+
kubectl wait -n createtree --for=condition=Complete --timeout=5m jobs createtree
46+
```
47+
48+
4. Check out the tree:
49+
```shell
50+
kubectl -n createtree get cm trillian-tree -ojsonpath='{.data.treeID}'
51+
```
52+
53+
You should see something like this:
54+
```shell
55+
vaikas@villes-mbp scaffolding % kubectl -n createtree get cm trillian-tree -ojsonpath='{.data.treeID}'
56+
5213139395739357930%
57+
```
58+
59+
5. You can then use the TreeID for example to run CTLog on
60+
top of newly created Trillian.
61+
62+
6. TODO: Add examples for talking to Trillian for other things.

‎examples/deployment/kubernetes/createtree/100-namespace.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
kind: Namespace
2+
apiVersion: v1
3+
metadata:
4+
name: createtree

‎examples/deployment/kubernetes/createtree/101-binding.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: Role
3+
metadata:
4+
namespace: createtree
5+
name: cm-operator
6+
rules:
7+
- apiGroups: [""] # "" indicates the core API group
8+
resources: ["configmaps"]
9+
resourceNames: ["trillian-tree"]
10+
verbs: ["get", "update"]
11+
---
12+
apiVersion: rbac.authorization.k8s.io/v1
13+
kind: RoleBinding
14+
metadata:
15+
name: role-cm-updater
16+
namespace: createtree
17+
roleRef:
18+
apiGroup: rbac.authorization.k8s.io
19+
kind: Role
20+
name: cm-operator
21+
subjects:
22+
- kind: ServiceAccount
23+
name: createtree
24+
namespace: createtree

‎examples/deployment/kubernetes/createtree/101-configmap.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: trillian-tree
5+
namespace: createtree
6+
data:
7+
__placeholder: |
8+
###################################################################
9+
# Just a placeholder so that reapplying this won't overwrite treeID
10+
# if it already exists. This caused grief, do not remove.
11+
###################################################################
12+

‎examples/deployment/kubernetes/createtree/101-service-account.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: createtree
5+
namespace: createtree

‎examples/deployment/kubernetes/createtree/300-createtree.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: createtree
5+
namespace: createtree
6+
spec:
7+
template:
8+
spec:
9+
serviceAccountName: createtree
10+
restartPolicy: Never
11+
containers:
12+
- name: createtree
13+
image: ko://github.com/google/trillian/cmd/createtree-k8s
14+
args: [
15+
"--namespace=createtree",
16+
"--configmap=trillian-tree",
17+
"--display_name=ctlogtree"
18+
]

‎examples/deployment/kubernetes/scaffolding/log-server/100-namespace.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
kind: Namespace
2+
apiVersion: v1
3+
metadata:
4+
name: trillian-system

0 commit comments

Comments
 (0)