From a4e394f9e56df57c6d37285d1a0e79126dd43bb1 Mon Sep 17 00:00:00 2001 From: madz Date: Wed, 11 May 2016 16:21:23 +0800 Subject: [PATCH 01/30] add pipeline and build delete --- api/build.go | 31 +++++++++++++++++++++++++++++++ api/pipeline.go | 27 +++++++++++++++++++++++++++ pipeline/build.go | 18 ++++++++++++++++++ pipeline/pipeline.go | 17 +++++++++++++++++ store/mc/mc.go | 22 ++++++++++++++++++++++ 5 files changed, 115 insertions(+) diff --git a/api/build.go b/api/build.go index 7a5fed3..d2f76f4 100644 --- a/api/build.go +++ b/api/build.go @@ -56,6 +56,16 @@ func (b *BuildResource) extend(ws *restful.WebService) { Param(ws.PathParameter("buildNumber", "build number").DataType("int")). Writes(ps.Build{}). Filter(requireAccessToken)) + + ws.Route(ws.DELETE("/{owner}/{repo}/builds/{buildNumber}").To(b.delete). + Doc("Remove build details"). + Operation("delete"). + Param(ws.PathParameter("owner", "repository owner name").DataType("string")). + Param(ws.PathParameter("repo", "repository name").DataType("string")). + Param(ws.PathParameter("buildNumber", "build number").DataType("int")). + Writes(ps.Build{}). + Filter(requireAccessToken)) + } func (b *BuildResource) create(req *restful.Request, res *restful.Response) { @@ -177,6 +187,27 @@ func (b *BuildResource) create(req *restful.Request, res *restful.Response) { res.WriteEntity(build) } +func (b *BuildResource) delete(req *restful.Request, res *restful.Response) { + + owner := req.PathParameter("owner") + repo := req.PathParameter("repo") + buildNumber := req.PathParameter("buildNumber") + pipeline, err := findPipeline(owner, repo, b.KVClient) + + if err != nil { + jsonError(res, http.StatusNotFound, err, fmt.Sprintf("Unable to find pipeline %s/%s", owner, repo)) + return + } + + build, err := findBuild(buildNumber, pipeline, b.KVClient) + if err != nil { + jsonError(res, http.StatusNotFound, err, fmt.Sprintf("Unable to find build %s for %s/%s", buildNumber, owner, repo)) + return + } + + build.Delete(b.KVClient, b.MinioClient) +} + func (b *BuildResource) list(req *restful.Request, res *restful.Response) { owner := req.PathParameter("owner") repo := req.PathParameter("repo") diff --git a/api/pipeline.go b/api/pipeline.go index 1c586ae..c7a7f96 100644 --- a/api/pipeline.go +++ b/api/pipeline.go @@ -63,6 +63,15 @@ func (p *PipelineResource) Register(container *restful.Container) { Filter(authenticate). Filter(requireAccessToken)) + ws.Route(ws.DELETE("/{owner}/{repo}").To(p.delete). + Doc("Delete pipeline"). + Operation("delete"). + Param(ws.PathParameter("owner", "repository owner name").DataType("string")). + Param(ws.PathParameter("repo", "repository name").DataType("string")). + Writes(ps.Pipeline{}). + Filter(authenticate). + Filter(requireAccessToken)) + ws.Route(ws.GET("/{owner}/{repo}/definition").To(p.definition). Doc("Get pipeline details of the repository"). Operation("definition"). @@ -126,6 +135,24 @@ func (p *PipelineResource) create(req *restful.Request, res *restful.Response) { res.WriteHeaderAndEntity(http.StatusCreated, pipeline) } +func (p *PipelineResource) delete(req *restful.Request, res *restful.Response) { + owner := req.PathParameter("owner") + repo := req.PathParameter("repo") + pipeline, err := findPipeline(owner, repo, p.KVClient) + if err != nil { + jsonError(res, http.StatusNotFound, err, fmt.Sprintf("Unable to find pipeline %s/%s", owner, repo)) + return + } + + if err := pipeline.DeletePipeline(p.KVClient, p.MinioClient); err != nil { + jsonError(res, http.StatusInternalServerError, err, fmt.Sprintf("Unable to delete pipeline %s/%s", owner, repo)) + return + } + + res.WriteHeader(http.StatusOK) + +} + func (p *PipelineResource) list(req *restful.Request, res *restful.Response) { pipelines, err := ps.FindAllPipelines(p.KVClient) if err != nil { diff --git a/pipeline/build.go b/pipeline/build.go index eb57932..9c131dd 100644 --- a/pipeline/build.go +++ b/pipeline/build.go @@ -11,6 +11,7 @@ import ( "github.com/AcalephStorage/kontinuous/kube" "github.com/AcalephStorage/kontinuous/notif" "github.com/AcalephStorage/kontinuous/store/kv" + "github.com/AcalephStorage/kontinuous/store/mc" ) // Build contains the details needed to run a build @@ -85,6 +86,23 @@ func getBuildSummary(path string, kvClient kv.KVClient) *BuildSummary { return b } +func (b *Build) Delete(kvClient kv.KVClient, mcClient *mc.MinioClient) (err error) { + path := fmt.Sprintf("%s%s/builds/%d", pipelineNamespace, b.Pipeline, b.Number) + buildsPrefix := fmt.Sprintf("pipelines/%s/builds/%d/", b.ID, b.Number) + bucket := "kontinuous" + + //remove build info from etcd + if err := kvClient.DeleteTree(path); err != nil { + return err + } + + //remove build {num} artifacts and logs from minio storage + if err := mcClient.DeleteTree(bucket, buildsPrefix); err != nil { + return err + } + return nil +} + // Save persists the build details to `etcd` func (b *Build) Save(kvClient kv.KVClient) (err error) { buildsPrefix := fmt.Sprintf("%s%s/builds", pipelineNamespace, b.Pipeline) diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index ec9c135..c3f023e 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -16,6 +16,7 @@ import ( "github.com/AcalephStorage/kontinuous/scm" "github.com/AcalephStorage/kontinuous/store/kv" + "github.com/AcalephStorage/kontinuous/store/mc" ) const ( @@ -307,6 +308,22 @@ func (p *Pipeline) Save(kvClient kv.KVClient) (err error) { return nil } +func (p *Pipeline) DeletePipeline(kvClient kv.KVClient, mcClient *mc.MinioClient) (err error) { + path := fmt.Sprintf("%s%s", pipelineNamespace, p.ID) + pipelinePrefix := fmt.Sprintf("pipelines/%s/", p.ID) + bucket := "kontinuous" + + if err := kvClient.DeleteTree(path); err != nil { + return err + } + + if err := mcClient.DeleteTree(bucket, pipelinePrefix); err != nil { + return err + } + return nil + +} + // Validate checks if the required values for a pipeline are present func (p *Pipeline) Validate() error { if p.Owner == "" { diff --git a/store/mc/mc.go b/store/mc/mc.go index 735529a..a2bd66a 100644 --- a/store/mc/mc.go +++ b/store/mc/mc.go @@ -36,6 +36,7 @@ func (mc *MinioClient) ListObjects(bucket, prefix string) ([]minio.ObjectInfo, e defer close(doneCh) // List all objects from a bucket-name with a matching prefix. + for object := range mc.client.ListObjects(bucket, prefix, true, doneCh) { if object.Err != nil { return result, object.Err @@ -52,3 +53,24 @@ func (mc *MinioClient) CopyLocally(bucket, object, file string) error { } return nil } + +func (mc *MinioClient) DeleteTree(bucket, prefix string) error { + doneCh := make(chan struct{}) + defer close(doneCh) + + for object := range mc.client.ListObjects(bucket, prefix, true, doneCh) { + err := mc.DeleteObject(bucket, object.Key) + if err != nil { + return err + } + } + return nil +} + +func (mc *MinioClient) DeleteObject(bucket, object string) error { + err := mc.client.RemoveObject(bucket, object) + if err != nil { + return err + } + return nil +} From 97da86652717f829555a76b63534aa03a286bc5c Mon Sep 17 00:00:00 2001 From: dexter Date: Wed, 11 May 2016 19:48:24 +0800 Subject: [PATCH 02/30] updated deploy script --- scripts/kontinuous-deploy | 128 +++++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 1 deletion(-) diff --git a/scripts/kontinuous-deploy b/scripts/kontinuous-deploy index 94132fd..1bc08fe 100755 --- a/scripts/kontinuous-deploy +++ b/scripts/kontinuous-deploy @@ -2,6 +2,7 @@ KONTINUOUS_SPECS_FILE=/tmp/kontinuous-specs.yml KONTINUOUS_RC_SPEC_FILE=/tmp/kontinuous-rc-spec.yml +KONTINUOUS_UI_RC_SPEC_FILE=/tmp/kontinuous-ui-rc-spec.yml read -r -d '' SECRET_DATA_TEMPLATE << \ "-----TEMPLATE-----" @@ -255,6 +256,69 @@ spec: readOnly: true -----TEMPLATE----- +read -r -d '' KONTINUOUS_UI_SVC_SPEC_TEMPLATE << \ +'-----TEMPLATE-----' +--- +kind: Service +apiVersion: v1 +metadata: + name: kontinuous-ui + namespace: namespace-data + labels: + app: kontinuous-ui + type: dashboard +spec: + type: LoadBalancer + selector: + app: kontinuous-ui + type: dashboard + ports: + - name: dashboard + port: 80 + targetPort: 5000 +-----TEMPLATE----- + + +read -r -d '' KONTINUOUS_UI_RC_SPEC_TEMPLATE << \ +'-----TEMPLATE-----' +--- +kind: ReplicationController +apiVersion: v1 +metadata: + labels: + app: kontinuous-ui + type: dashboard + name: kontinuous-ui + namespace: namespace-data +spec: + replicas: 1 + selector: + app: kontinuous-ui + type: dashboard + template: + metadata: + name: kontinuous-ui + namespace: namespace-data + labels: + app: kontinuous-ui + type: dashboard + spec: + containers: + - name: kontinuous-ui + image: quay.io/acaleph/kontinuous-ui:0.1.0 + imagePullPolicy: Always + env: + - name: GITHUB_CLIENT_CALLBACK + value: http://kontinuous-ui-ip + - name: GITHUB_CLIENT_ID + value: gh-client-id # 65f8775ac3fc2e7e27eb + - name: KONTINUOUS_API_URL + value: http://kontinuous-ip:8080 + ports: + - name: dashboard + containerPort: 5000 +-----TEMPLATE----- + read -r -d '' HELP_MESSAGE << \ '-----HELP-----' ----------------- @@ -273,6 +337,7 @@ Options: --auth-secret base64 encoded auth secret to be used for JWT authentication --s3-access-key s3 access key --s3-secret-key s3 secret key + --github-client-id github client ID for the registered kontinuous app --delete delete create k8s resources -----HELP----- @@ -362,6 +427,29 @@ function create_kontinuous_rc_spec() { echo "$kontinuous_spec" > $KONTINUOUS_RC_SPEC_FILE } +function create_kontinuous_ui_spec() { + local namespace="$1"; shift + + local kontinuous_ui_spec="${KONTINUOUS_UI_SVC_SPEC_TEMPLATE/namespace-data/$namespace}" + + echo "$kontinuous_ui_spec" >> $KONTINUOUS_SPECS_FILE +} + +function create_kontinuous_ui_rc_spec() { + local namespace="$1"; shift + local kontinuous_ip="$1"; shift + local kontinuous_ui_ip="$1"; shift + local gh_client_id="$1"; shift + + local kontinuous_ui_rc_spec="${KONTINUOUS_UI_RC_SPEC_TEMPLATE/namespace-data/$namespace}" + kontinuous_ui_rc_spec="${kontinuous_ui_rc_spec/namespace-data/$namespace}" + kontinuous_ui_rc_spec="${kontinuous_ui_rc_spec/kontinuous-ip/$kontinuous_ip}" + kontinuous_ui_rc_spec="${kontinuous_ui_rc_spec/kontinuous-ui-ip/$kontinuous_ui_ip}" + kontinuous_ui_rc_spec="${kontinuous_ui_rc_spec/gh-client-id/$gh_client_id}" + + echo "$kontinuous_ui_rc_spec" > $KONTINUOUS_UI_RC_SPEC_FILE +} + function fetch_kontinuous_ip() { local namespace="$1"; shift @@ -379,13 +467,36 @@ function fetch_kontinuous_ip() { echo "$ip" } +function fetch_kontinuous_ui_ip() { + local namespace="$1"; shift + + local ip='' + while [[ "$ip" == '' || "$ip" == '' ]]; do + data=$(kubectl get svc kontinuous-ui --namespace="$namespace" -o template --template="{{.status.loadBalancer.ingress}}") + if [[ "$data" != '' && "$data" != '' ]]; then + ip=${data/[map[ip:/} + ip=${ip/]]/} + else + sleep 5 + fi + done + + echo "$ip" +} + function create_kontinuous_rc_resource() { kubectl create -f $KONTINUOUS_RC_SPEC_FILE } +function create_kontinuous_ui_rc_resource() { + kubectl create -f $KONTINUOUS_UI_RC_SPEC_FILE +} + + function remove_resources() { kubectl delete -f $KONTINUOUS_SPECS_FILE kubectl delete -f $KONTINUOUS_RC_SPEC_FILE + kubectl delete -f $KONTINUOUS_UI_RC_SPEC_FILE } function main() { @@ -414,8 +525,14 @@ function main() { shift ;; + --github-client-id) + local gh_client_id="$2" + shift + ;; + --delete) remove_resources + exit 0 ;; --help) @@ -435,12 +552,21 @@ function main() { create_minio_spec "$namespace" "$s3_access_key" "$s3_secret_key" create_registry_spec "$namespace" create_kontinuous_spec "$namespace" - # create_kontinuous_resources + create_kontinuous_ui_spec "$namespace" + create_kontinuous_resources echo 'Waiting for Kontinuous IP...' local kontinuous_ip=$(fetch_kontinuous_ip "$namespace") create_kontinuous_rc_spec "$namespace" "$kontinuous_ip" create_kontinuous_rc_resource + + echo 'Waiting for Kontinuous UI IP...' + local kontinuous_ui_ip=$(fetch_kontinuous_ui_ip "$namespace") + create_kontinuous_ui_rc_spec "$namespace" "$kontinuous_ip" "$kontinuous_ui_ip" "$gh_client_id" + create_kontinuous_ui_rc_resource + + echo + echo '---Kontinuous Deployed---' } main $@ From 6c467c83eb11c33e6c5df7b12800d380bf8199c3 Mon Sep 17 00:00:00 2001 From: madz Date: Thu, 12 May 2016 12:28:40 +0800 Subject: [PATCH 03/30] fix path and prefixes --- pipeline/build.go | 2 +- pipeline/pipeline.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipeline/build.go b/pipeline/build.go index 9c131dd..1a152af 100644 --- a/pipeline/build.go +++ b/pipeline/build.go @@ -88,7 +88,7 @@ func getBuildSummary(path string, kvClient kv.KVClient) *BuildSummary { func (b *Build) Delete(kvClient kv.KVClient, mcClient *mc.MinioClient) (err error) { path := fmt.Sprintf("%s%s/builds/%d", pipelineNamespace, b.Pipeline, b.Number) - buildsPrefix := fmt.Sprintf("pipelines/%s/builds/%d/", b.ID, b.Number) + buildsPrefix := fmt.Sprintf("pipelines/%s/builds/%d", b.ID, b.Number) bucket := "kontinuous" //remove build info from etcd diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index c3f023e..1d34a7a 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -309,8 +309,8 @@ func (p *Pipeline) Save(kvClient kv.KVClient) (err error) { } func (p *Pipeline) DeletePipeline(kvClient kv.KVClient, mcClient *mc.MinioClient) (err error) { - path := fmt.Sprintf("%s%s", pipelineNamespace, p.ID) - pipelinePrefix := fmt.Sprintf("pipelines/%s/", p.ID) + path := fmt.Sprintf("%s%s", pipelineNamespace, p.fullName()) + pipelinePrefix := fmt.Sprintf("pipelines/%s", p.ID) bucket := "kontinuous" if err := kvClient.DeleteTree(path); err != nil { From 660dc2c8c84569af91407823f5a274e79c89f52a Mon Sep 17 00:00:00 2001 From: madz Date: Thu, 12 May 2016 13:02:55 +0800 Subject: [PATCH 04/30] use pipeline id instead of build id --- api/build.go | 2 +- pipeline/build.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/build.go b/api/build.go index d2f76f4..d2c78b9 100644 --- a/api/build.go +++ b/api/build.go @@ -205,7 +205,7 @@ func (b *BuildResource) delete(req *restful.Request, res *restful.Response) { return } - build.Delete(b.KVClient, b.MinioClient) + build.Delete(pipeline.ID, b.KVClient, b.MinioClient) } func (b *BuildResource) list(req *restful.Request, res *restful.Response) { diff --git a/pipeline/build.go b/pipeline/build.go index 1a152af..ffa958d 100644 --- a/pipeline/build.go +++ b/pipeline/build.go @@ -86,9 +86,9 @@ func getBuildSummary(path string, kvClient kv.KVClient) *BuildSummary { return b } -func (b *Build) Delete(kvClient kv.KVClient, mcClient *mc.MinioClient) (err error) { +func (b *Build) Delete(pipelinesID string, kvClient kv.KVClient, mcClient *mc.MinioClient) (err error) { path := fmt.Sprintf("%s%s/builds/%d", pipelineNamespace, b.Pipeline, b.Number) - buildsPrefix := fmt.Sprintf("pipelines/%s/builds/%d", b.ID, b.Number) + buildsPrefix := fmt.Sprintf("pipelines/%s/builds/%d", pipelinesID, b.Number) bucket := "kontinuous" //remove build info from etcd From 3596a747607e3ecdc2c95cec794a8e8cd0491153 Mon Sep 17 00:00:00 2001 From: madz Date: Thu, 12 May 2016 16:22:39 +0800 Subject: [PATCH 05/30] add delete pipeline and build support for kontinuous-cli --- cli/cli.go | 57 ++++++++++++++++++++++++++++++++++++++++++++++ cli/request/api.go | 18 +++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/cli/cli.go b/cli/cli.go index 2b5a449..50d7818 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -94,6 +94,31 @@ func main() { }, }, }, + { + Name: "delete", + Subcommands: []cli.Command{ + { + Name: "pipeline", + Usage: "delete pipeline", + ArgsUsage: "", + Before: requireNameArg, + Action: deletePipeline, + }, + { + Name: "build", + Usage: "delete build", + ArgsUsage: "", + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "build, b", + Usage: "build number, if not provided will not proceed deletion", + }, + }, + Before: requireNameArg, + Action: deleteBuild, + }, + }, + }, } app.Run(os.Args) } @@ -264,3 +289,35 @@ func createBuild(c *cli.Context) { fmt.Printf("building pipeline %s/%s ", owner, repo) } } + +func deletePipeline(c *cli.Context) { + config, err := apiReq.GetConfigFromFile(c.GlobalString("conf")) + if err != nil { + os.Exit(1) + } + pipelineName := c.Args().First() + err = config.DeletePipeline(http.DefaultClient, pipelineName) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + fmt.Printf("pipeline %s successfully deleted.\n", pipelineName) +} + +func deleteBuild(c *cli.Context) { + config, err := apiReq.GetConfigFromFile(c.GlobalString("conf")) + if err != nil { + os.Exit(1) + } + + pipelineName := c.Args().First() + buildNum := c.String("build") + err = config.DeleteBuild(http.DefaultClient, pipelineName, buildNum) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + fmt.Printf("pipeline %s build #%s successfully deleted.\n", pipelineName, buildNum) +} diff --git a/cli/request/api.go b/cli/request/api.go index 049b963..9413ce9 100644 --- a/cli/request/api.go +++ b/cli/request/api.go @@ -217,6 +217,24 @@ func (c *Config) CreateBuild(client *http.Client, owner, repo string) error { return nil } +func (c *Config) DeletePipeline(client *http.Client, pipelineName string) error { + endpoint := fmt.Sprintf("/api/v1/pipelines/%s", pipelineName) + _, err := c.sendAPIRequest(client, "DELETE", endpoint, nil) + if err != nil { + return err + } + return nil +} + +func (c *Config) DeleteBuild(client *http.Client, pipelineName string, buildNumber string) error { + endpoint := fmt.Sprintf("/api/v1/pipelines/%s/builds/%v", pipelineName, buildNumber) + _, err := c.sendAPIRequest(client, "DELETE", endpoint, nil) + if err != nil { + return err + } + return nil +} + func (c *Config) validate() error { missing := []string{} if len(c.Host) == 0 { From 98c8438772adbbb2d9c2adb410f3b98519a0a3f9 Mon Sep 17 00:00:00 2001 From: madz Date: Fri, 13 May 2016 17:47:17 +0800 Subject: [PATCH 06/30] add integrate deploy in cli (wip) --- cli/cli.go | 67 ++++++++ cli/deploy.go | 420 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 487 insertions(+) create mode 100644 cli/deploy.go diff --git a/cli/cli.go b/cli/cli.go index 50d7818..73e7f2a 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -119,6 +119,37 @@ func main() { }, }, }, + { + Name: "deploy", + Usage: "deploy kontinuous app in the cluster", + Subcommands: []cli.Command{ + { + Name: "remove", + Usage: "remove kontinuous app in the cluster", + Action: removeDeployedApp, + }, + }, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "namespace", + Usage: "Required, kontinuous namespace", + }, + cli.StringFlag{ + Name: "accesskey", + Usage: "Required, s3 access key", + }, + cli.StringFlag{ + Name: "secretkey", + Usage: "Required, s3 secret key", + }, + cli.StringFlag{ + Name: "authcode", + Usage: "Required, jwt authorization code", + }, + }, + + Action: deployApp, + }, } app.Run(os.Args) } @@ -321,3 +352,39 @@ func deleteBuild(c *cli.Context) { fmt.Printf("pipeline %s build #%s successfully deleted.\n", pipelineName, buildNum) } + +func deployApp(c *cli.Context) { + namespace := c.String("namespace") + accessKey := c.String("accesskey") + secretKey := c.String("secretkey") + authCode := c.String("authcode") + + missingFields := false + if namespace == "" || accessKey == "" || secretKey == "" || authCode == "" { + fmt.Println("missing required fields!") + missingFields = true + + } + + if !missingFields { + err := DeployKontinuous(namespace, accessKey, secretKey, authCode) + if err != nil { + fmt.Println("Oops something went wrong. Unable to deploy kontinuous.") + fmt.Println(err) + os.Exit(1) + } + fmt.Println("Success! Kontinuous is now deployed in the cluster.") + } +} + +func removeDeployedApp(c *cli.Context) { + err := RemoveResources() + + if err != nil { + fmt.Println("Oops something went wrong. Unable to remove kontinuous.") + fmt.Println(err) + os.Exit(1) + } + + fmt.Println("Success! Kontinuous app has been removed from the cluster. ") +} diff --git a/cli/deploy.go b/cli/deploy.go new file mode 100644 index 0000000..562e816 --- /dev/null +++ b/cli/deploy.go @@ -0,0 +1,420 @@ +package main + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "strings" + "text/template" + "time" +) + +var secretData = ` +{ + "AuthSecret": "{{.AuthCode}}", + "S3SecretKey": "{{.SecretKey}}", + "S3AccessKey": "{{.AccessKey}}" +} + +` + +var minio = ` +--- +kind: Service +apiVersion: v1 +metadata: + name: minio + namespace: {{.Namespace}} + labels: + app: minio + type: object-store +spec: + selector: + app: minio + type: object-store + ports: + - name: service + port: 9000 + targetPort: 9000 +--- +kind: ReplicationController +apiVersion: v1 +metadata: + name: minio + namespace: {{.Namespace}} + labels: + app: minio + type: object-store +spec: + replicas: 1 + selector: + app: minio + type: object-store + template: + metadata: + name: minio + labels: + app: minio + type: object-store + spec: + containers: + - name: minio + image: minio/minio:latest + imagePullPolicy: Always + env: + - name: MINIO_ACCESS_KEY + value: {{.AccessKey}} + - name: MINIO_SECRET_KEY + value: {{.SecretKey}} + ports: + - name: service + containerPort: 9000 + livenessProbe: + tcpSocket: + port: 9000 + timeoutSeconds: 1 +` + +var secret = ` + +--- +kind: Secret +apiVersion: v1 +metadata: + name: kontinuous-secrets + namespace: {{.Namespace}} +data: + kontinuous-secrets: {{.SecretData}} + +` + +var etcd = ` +--- +kind: Service +apiVersion: v1 +metadata: + name: etcd + namespace: {{.Namespace}} + labels: + app: etcd + type: kv +spec: + selector: + app: etcd + type: kv + ports: + - name: api + port: 2379 + targetPort: 2379 +--- +kind: ReplicationController +apiVersion: v1 +metadata: + name: etcd + namespace: {{.Namespace}} + labels: + app: etcd + type: kv +spec: + replicas: 1 + selector: + app: etcd + type: kv + template: + metadata: + labels: + app: etcd + type: kv + spec: + containers: + - name: etcd + image: quay.io/coreos/etcd:v2.2.2 + imagePullPolicy: Always + args: + - --listen-client-urls + - http://0.0.0.0:2379 + - --advertise-client-urls + - http://0.0.0.0:2379 + ports: + - name: api + containerPort: 2379 +` + +var registry = ` +--- +kind: Service +apiVersion: v1 +metadata: + name: registry + namespace: {{.Namespace}} + labels: + app: registry + type: storage +spec: + selector: + app: registry + type: storage + ports: + - name: service + port: 5000 + targetPort: 5000 +--- +kind: ReplicationController +apiVersion: v1 +metadata: + name: registry + namespace: {{.Namespace}} + labels: + app: registry + type: storage +spec: + replicas: 1 + selector: + app: registry + type: storage + template: + metadata: + name: registry + namespace: {{.Namespace}} + labels: + app: registry + type: storage + spec: + containers: + - name: registry + image: registry:2 + ports: + - name: service + containerPort: 5000 + +` + +var kontinuousService = ` +--- +kind: Service +apiVersion: v1 +metadata: + name: kontinuous + namespace: {{.Namespace}} + labels: + app: kontinuous + type: ci-cd +spec: + type: LoadBalancer + selector: + app: kontinuous + type: ci-cd + ports: + - name: api + port: 8080 + targetPort: 3005 +` + +var kontinuousRC = ` +--- +kind: ReplicationController +apiVersion: v1 +metadata: + name: kontinuous + namespace: {{.Namespace}} + labels: + app: kontinuous + type: ci-cd +spec: + replicas: 1 + selector: + app: kontinuous + type: ci-cd + template: + metadata: + labels: + app: kontinuous + type: ci-cd + spec: + volumes: + - name: kontinuous-secrets + secret: + secretName: kontinuous-secrets + containers: + - name: kontinuous + image: quay.io/acaleph/kontinuous:latest + imagePullPolicy: Always + env: + - name: KV_ADDRESS + value: etcd:2379 + - name: S3_URL + value: http://minio:9000 + - name: KONTINUOUS_URL + value: http://{{.KontinuousIP}}:8080 + - name: INTERNAL_REGISTRY + value: registry:5000 + ports: + - name: api + containerPort: 3005 + volumeMounts: + - mountPath: /.secret + name: kontinuous-secrets + readOnly: true +` + +type Deploy struct { + Namespace string + AccessKey string + SecretKey string + AuthCode string + SecretData string + KontinuousIP string +} + +const ( + KONTINUOUS_SPECS_FILE = "/tmp/kontinuous-specs.yml" + KONTINUOUS_RC_SPEC_FILE = "/tmp/kontinuous-rc-spec.yml" +) + +func generateResource(templateStr string, deploy *Deploy) (string, error) { + + template := template.New("kontinuous Template") + template, _ = template.Parse(templateStr) + var b bytes.Buffer + + err := template.Execute(&b, deploy) + + if err != nil { + fmt.Println(err.Error()) + } + + return b.String(), nil + +} + +func saveToFile(path string, data ...string) error { + var _, err = os.Stat(path) + var file *os.File + + if os.IsNotExist(err) { + file, _ = os.Create(path) + defer file.Close() + } + + file, err = os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0644) + if err != nil { + fmt.Println(err.Error()) + return err + } + + defer file.Close() + for _, dataStr := range data { + _, err = file.WriteString(dataStr) + + if err != nil { + fmt.Println(err.Error()) + return err + } + } + + err = file.Sync() + if err != nil { + fmt.Println(err.Error()) + return err + } + + return nil +} + +func encryptSecret(secret string) (string, error) { + cmd := fmt.Sprintf(`echo -n "%s" | openssl base64 | tr -d "\n"`, secret) + out, err := exec.Command("bash", "-c", cmd).Output() + if err != nil { + return "", err + } + return string(out), nil + +} + +func createKontinuousResouces(path string) error { + cmd := fmt.Sprintf("kubectl create -f %s", path) + _, err := exec.Command("bash", "-c", cmd).Output() + if err != nil { + return err + } + return nil +} + +func deleteKontinuousResources(path string) error { + cmd := fmt.Sprintf("kubectl delete -f %s", path) + _, err := exec.Command("bash", "-c", cmd).Output() + if err != nil { + return err + } + return nil +} + +func fetchKontinuousIP(namespace string) (string, error) { + var ip string + + cmd := fmt.Sprintf(`kubectl get svc kontinuous --namespace=%s -o template --template="{{.status.loadBalancer.ingress}}"`, namespace) + for len(ip) == 0 { + out, err := exec.Command("bash", "-c", cmd).Output() + if err != nil { + return "", err + } + + outStr := string(out) + if !strings.Contains(outStr, "") && !strings.Contains(outStr, "") { + ipStr := strings.TrimPrefix(outStr, "[map[ip:") + ip = strings.TrimSuffix(ipStr, "]]") + } else { + time.Sleep(5 * time.Second) + } + } + return ip, nil +} + +func RemoveResources() error { + err := deleteKontinuousResources(KONTINUOUS_SPECS_FILE) + if err != nil { + return err + } + err = deleteKontinuousResources(KONTINUOUS_RC_SPEC_FILE) + if err != nil { + return err + } + return nil +} + +func DeployKontinuous(namespace, accesskey, secretkey, authcode string) error { + deploy := Deploy{ + Namespace: namespace, + AccessKey: accesskey, + SecretKey: secretkey, + AuthCode: authcode, + } + sData, _ := generateResource(secretData, &deploy) + encryptedSecret, _ := encryptSecret(sData) + deploy.SecretData = encryptedSecret + secret, _ := generateResource(secret, &deploy) + minioStr, _ := generateResource(minio, &deploy) + etcdStr, _ := generateResource(etcd, &deploy) + registryStr, _ := generateResource(registry, &deploy) + kontinuousSvcStr, _ := generateResource(kontinuousService, &deploy) + + //save string in a file + saveToFile(KONTINUOUS_SPECS_FILE, secret, minioStr, etcdStr, registryStr, kontinuousSvcStr) + err := createKontinuousResouces(KONTINUOUS_SPECS_FILE) + + if err != nil { + return err + } + + ip, _ := fetchKontinuousIP("acaleph") + deploy.KontinuousIP = ip + kontinuousRcStr, _ := generateResource(kontinuousRC, &deploy) + saveToFile(KONTINUOUS_RC_SPEC_FILE, kontinuousRcStr) + err = createKontinuousResouces(KONTINUOUS_RC_SPEC_FILE) + + if err != nil { + return err + } + return nil +} From 336e77973d88f033263cd67048eec9fcb25350b5 Mon Sep 17 00:00:00 2001 From: madz Date: Fri, 13 May 2016 18:42:57 +0800 Subject: [PATCH 07/30] add ui svc and rc in deploy --- cli/cli.go | 9 ++++-- cli/deploy.go | 86 +++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index 73e7f2a..f1d5aff 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -146,6 +146,10 @@ func main() { Name: "authcode", Usage: "Required, jwt authorization code", }, + cli.StringFlag{ + Name: "clientid", + Usage: "Required, github client id", + }, }, Action: deployApp, @@ -358,16 +362,17 @@ func deployApp(c *cli.Context) { accessKey := c.String("accesskey") secretKey := c.String("secretkey") authCode := c.String("authcode") + clientId := c.String("clientid") missingFields := false - if namespace == "" || accessKey == "" || secretKey == "" || authCode == "" { + if namespace == "" || accessKey == "" || secretKey == "" || authCode == "" || clientId == "" { fmt.Println("missing required fields!") missingFields = true } if !missingFields { - err := DeployKontinuous(namespace, accessKey, secretKey, authCode) + err := DeployKontinuous(namespace, accessKey, secretKey, authCode, clientId) if err != nil { fmt.Println("Oops something went wrong. Unable to deploy kontinuous.") fmt.Println(err) diff --git a/cli/deploy.go b/cli/deploy.go index 562e816..9f91e59 100644 --- a/cli/deploy.go +++ b/cli/deploy.go @@ -258,6 +258,70 @@ spec: readOnly: true ` +var dashboardSvc = ` +--- +apiVersion: v1 +kind: Service +metadata: + labels: + service: kontinuous-ui + type: dashboard + name: kontinuous-ui + namespace: {{.Namespace}} +spec: + ports: + - name: dashboard + nodePort: 30345 + port: 5000 + protocol: TCP + targetPort: 5000 + selector: + app: kontinuous-ui + type: dashboard + type: LoadBalancer + +` + +var dashboardRc = ` +--- +apiVersion: v1 +kind: ReplicationController +metadata: + labels: + app: kontinuous-ui + type: dashboard + name: kontinuous-ui + namespace: {{.Namespace}} +spec: + replicas: 1 + selector: + app: kontinuous-ui + type: dashboard + template: + metadata: + labels: + app: kontinuous-ui + type: dashboard + name: kontinuous-ui + namespace: {{.Namespace}} + spec: + containers: + - env: + - name: GITHUB_CLIENT_CALLBACK + value: http://{{.DashboardIP}}:5000 + - name: GITHUB_CLIENT_ID + value: {{.GHClient}} + - name: KONTINUOUS_API_URL + value: http://{{.KontinuousIP}}:8080 + image: quay.io/acaleph/kontinuous-ui:latest + imagePullPolicy: Always + name: kontinuous-ui + ports: + - containerPort: 5000 + name: dashboard + +` + type Deploy struct { Namespace string AccessKey string @@ -265,6 +329,8 @@ type Deploy struct { AuthCode string SecretData string KontinuousIP string + DashboardIP string + GHClient string } const ( @@ -350,10 +416,10 @@ func deleteKontinuousResources(path string) error { return nil } -func fetchKontinuousIP(namespace string) (string, error) { +func fetchKontinuousIP(serviceName, namespace string) (string, error) { var ip string - cmd := fmt.Sprintf(`kubectl get svc kontinuous --namespace=%s -o template --template="{{.status.loadBalancer.ingress}}"`, namespace) + cmd := fmt.Sprintf(`kubectl get svc %s --namespace=%s -o template --template="{{.status.loadBalancer.ingress}}"`, serviceName, namespace) for len(ip) == 0 { out, err := exec.Command("bash", "-c", cmd).Output() if err != nil { @@ -383,12 +449,13 @@ func RemoveResources() error { return nil } -func DeployKontinuous(namespace, accesskey, secretkey, authcode string) error { +func DeployKontinuous(namespace, accesskey, secretkey, authcode, clientid string) error { deploy := Deploy{ Namespace: namespace, AccessKey: accesskey, SecretKey: secretkey, AuthCode: authcode, + GHClient: clientid, } sData, _ := generateResource(secretData, &deploy) encryptedSecret, _ := encryptSecret(sData) @@ -398,23 +465,30 @@ func DeployKontinuous(namespace, accesskey, secretkey, authcode string) error { etcdStr, _ := generateResource(etcd, &deploy) registryStr, _ := generateResource(registry, &deploy) kontinuousSvcStr, _ := generateResource(kontinuousService, &deploy) + dashboardSvcStr, _ := generateResource(dashboardSvc, &deploy) //save string in a file - saveToFile(KONTINUOUS_SPECS_FILE, secret, minioStr, etcdStr, registryStr, kontinuousSvcStr) + saveToFile(KONTINUOUS_SPECS_FILE, secret, minioStr, etcdStr, registryStr, kontinuousSvcStr, dashboardSvcStr) err := createKontinuousResouces(KONTINUOUS_SPECS_FILE) if err != nil { return err } - ip, _ := fetchKontinuousIP("acaleph") + ip, _ := fetchKontinuousIP("kontinuous", deploy.Namespace) + dashboardIp, _ := fetchKontinuousIP("kontinuous-ui", deploy.Namespace) + deploy.DashboardIP = dashboardIp deploy.KontinuousIP = ip + kontinuousRcStr, _ := generateResource(kontinuousRC, &deploy) - saveToFile(KONTINUOUS_RC_SPEC_FILE, kontinuousRcStr) + dashboardRcStr, _ := generateResource(dashboardRc, &deploy) + + saveToFile(KONTINUOUS_RC_SPEC_FILE, kontinuousRcStr, dashboardRcStr) err = createKontinuousResouces(KONTINUOUS_RC_SPEC_FILE) if err != nil { return err } + return nil } From 8f86277ff8dd57c3a1daef1eb6593899162abcc6 Mon Sep 17 00:00:00 2001 From: Stefany Serino Date: Tue, 17 May 2016 00:53:36 +0800 Subject: [PATCH 08/30] Rename scm client method from GetContents to GetFileContent to make way for GetContents that returns type RepositoryContent --- api/pipeline.go | 3 +- pipeline/common_test.go | 104 +++++++++++++++++++++------------------- pipeline/pipeline.go | 2 +- pipeline/stage.go | 2 +- scm/client.go | 8 +++- scm/github/github.go | 27 +++++++++-- 6 files changed, 89 insertions(+), 57 deletions(-) diff --git a/api/pipeline.go b/api/pipeline.go index 1c586ae..5515f6c 100644 --- a/api/pipeline.go +++ b/api/pipeline.go @@ -4,12 +4,11 @@ import ( "fmt" "net/http" - "github.com/emicklei/go-restful" - "github.com/AcalephStorage/kontinuous/kube" ps "github.com/AcalephStorage/kontinuous/pipeline" "github.com/AcalephStorage/kontinuous/store/kv" "github.com/AcalephStorage/kontinuous/store/mc" + "github.com/emicklei/go-restful" ) // PipelineResource defines the endpoints of a Pipeline diff --git a/pipeline/common_test.go b/pipeline/common_test.go index 56c9c42..d157c0f 100644 --- a/pipeline/common_test.go +++ b/pipeline/common_test.go @@ -22,6 +22,53 @@ type ( } ) +var validyamlSpec = ` +apiVersion: v1alpha1 +kind: Pipeline +metadata: + name: my-pipeline + namespace: acaleph +spec: + selector: + matchLabels: + app: my-pipeline-infra # can be used as a selector for finding infra launched during a build? + template: # taken from Job spec... needed? + metadata: + name: my-pipeline + labels: + app: my-pipeline + image: acaleph/deploy-base # Overridable? + stages: + - name: Test Infra + type: command + params: + command: ["test_infra.sh", "--hack-the-gibson"] + env: # env vars + - name: MYVAR + value: my value + artifact_paths: # sent to minio? + - "logs/**/*" + - "coverage/**/*" + timeout: 60 # kills after X minutes? + - name: "Have you finished testing?" # friendly name + type: block # waits for the user to approve + - name: Teardown Deploy local test infra + type: deploy_cleanup # stop + selector: + build: my-val-stage-1 # selects build to stop + - name: "Do you want to Publish to Production?" # friendly name + type: block # waits for the user to approve + - name: Deploy production + type: deploy + params: + template: production.yaml # DM template + properties: # DM properties + external_service: true + replicas: 3 + labels: # labels to cleanup build + state: canary + ` + func setupStore() kv.KVClient { return &MockKVClient{data: map[string]string{}} } @@ -172,61 +219,20 @@ func (s MockSCMClient) CreateKey(owner, repo, key, title string) error { return nil } -func (s MockSCMClient) GetContents(owner, repo, content, ref string) ([]byte, bool) { +func (s MockSCMClient) GetFileContent(owner, repo, path, ref string) ([]byte, bool) { if !s.success { return nil, false } - - validyamlSpec = ` -apiVersion: v1alpha1 -kind: Pipeline -metadata: - name: my-pipeline - namespace: acaleph -spec: - selector: - matchLabels: - app: my-pipeline-infra # can be used as a selector for finding infra launched during a build? - template: # taken from Job spec... needed? - metadata: - name: my-pipeline - labels: - app: my-pipeline - image: acaleph/deploy-base # Overridable? - stages: - - name: Test Infra - type: command - params: - command: ["test_infra.sh", "--hack-the-gibson"] - env: # env vars - - name: MYVAR - value: my value - artifact_paths: # sent to minio? - - "logs/**/*" - - "coverage/**/*" - timeout: 60 # kills after X minutes? - - name: "Have you finished testing?" # friendly name - type: block # waits for the user to approve - - name: Teardown Deploy local test infra - type: deploy_cleanup # stop - selector: - build: my-val-stage-1 # selects build to stop - - name: "Do you want to Publish to Production?" # friendly name - type: block # waits for the user to approve - - name: Deploy production - type: deploy - params: - template: production.yaml # DM template - properties: # DM properties - external_service: true - replicas: 3 - labels: # labels to cleanup build - state: canary - ` - return []byte(validyamlSpec), true } +func (s MockSCMClient) GetContents(owner, repo, path, ref string) (*scm.RepositoryContent, bool) { + if !s.success { + return nil, false + } + return &scm.RepositoryContent{validyamlSpec}, true +} + func (s MockSCMClient) ListRepositories(user string) ([]*scm.Repository, error) { return nil, nil } diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index ec9c135..640b98c 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -354,7 +354,7 @@ func (p *Pipeline) Validate() error { // Definition retrieves the pipeline definition from a given reference func (p *Pipeline) Definition(ref string, c scm.Client) (*Definition, error) { - file, ok := c.GetContents(p.Owner, p.Repo, PipelineYAML, ref) + file, ok := c.GetFileContent(p.Owner, p.Repo, PipelineYAML, ref) if !ok { return nil, fmt.Errorf("%s not found for %s/%s on %s", PipelineYAML, diff --git a/pipeline/stage.go b/pipeline/stage.go index f55e230..5161b6b 100644 --- a/pipeline/stage.go +++ b/pipeline/stage.go @@ -149,7 +149,7 @@ func (s *Stage) Deploy(p *Pipeline, b *Build, c scm.Client) error { ref = b.Branch } - file, ok := c.GetContents(p.Owner, p.Repo, deployFile, b.Commit) + file, ok := c.GetFileContent(p.Owner, p.Repo, deployFile, b.Commit) if !ok { return fmt.Errorf("%s not found for %s/%s on %s", diff --git a/scm/client.go b/scm/client.go index 8722f4d..99cd834 100644 --- a/scm/client.go +++ b/scm/client.go @@ -44,7 +44,8 @@ type Client interface { CreateHook(owner, repo, callback string, events []string) error CreateKey(owner, repo, key, title string) error CreateStatus(owner, repo, sha string, stageId int, stageName, state string) error - GetContents(owner, repo, content, ref string) ([]byte, bool) + GetFileContent(owner, repo, path, ref string) ([]byte, bool) + GetContents(owner, repo, path, ref string) (*RepositoryContent, bool) GetRepository(owner, repo string) (*Repository, bool) ListRepositories(user string) ([]*Repository, error) ParseHook(payload []byte, event string) (*Hook, error) @@ -62,6 +63,11 @@ type Repository struct { Permissions map[string]bool `json:"-"` } +type RepositoryContent struct { + Content *string `json:"content"` + SHA *string `json:"sha,omitempty"` +} + // IsAdmin determines if the scoped user has admin rights for the repository func (r *Repository) IsAdmin() bool { return r.Permissions["admin"] diff --git a/scm/github/github.go b/scm/github/github.go index 299ba80..5588526 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -74,11 +74,11 @@ func (gc *Client) CreateStatus(owner, repo, ref string, stageId int, stageName, return nil } -// GetContents fetches a file from the given commit or branch -func (gc *Client) GetContents(owner, repo, content, ref string) ([]byte, bool) { +// GetFileContent fetches a file from the given commit or branch +func (gc *Client) GetFileContent(owner, repo, path, ref string) ([]byte, bool) { file, _, _, err := gc.client().Repositories.GetContents(owner, repo, - content, + path, &github.RepositoryContentGetOptions{ref}) if err != nil { return nil, false @@ -92,6 +92,27 @@ func (gc *Client) GetContents(owner, repo, content, ref string) ([]byte, bool) { return decoded, true } +// GetContents gets the metadata and content of a file from the given commit or branch +func (gc *Client) GetContents(owner, repo, path, ref string) (*scm.RepositoryContent, bool) { + file, _, _, err := gc.client().Repositories.GetContents(owner, + repo, + path, + &github.RepositoryContentGetOptions{ref}) + if err != nil { + return nil, false + } + + decoded, err := file.GetContent() + if err != nil { + return nil, false + } + + return &scm.RepositoryContent{ + Content: &decoded, + SHA: file.SHA, + }, true +} + // GetRepository fetches repository details from GitHub func (gc *Client) GetRepository(owner, name string) (*scm.Repository, bool) { data, _, err := gc.client().Repositories.Get(owner, name) From 2ee01c7f7d2c5922a48c8e1a48a7aaefe4a2803f Mon Sep 17 00:00:00 2001 From: Stefany Serino Date: Tue, 17 May 2016 13:14:27 +0800 Subject: [PATCH 09/30] Add endpoint for updating definition file (by committing diff) --- api/pipeline.go | 48 ++++++++++++++++++++++++++++++++---- pipeline/common_test.go | 10 ++++++-- pipeline/definition_test.go | 49 +------------------------------------ scm/client.go | 6 +++-- scm/github/github.go | 28 +++++++++++++++------ 5 files changed, 76 insertions(+), 65 deletions(-) diff --git a/api/pipeline.go b/api/pipeline.go index 5515f6c..2947932 100644 --- a/api/pipeline.go +++ b/api/pipeline.go @@ -2,6 +2,9 @@ package api import ( "fmt" + + "encoding/base64" + "io/ioutil" "net/http" "github.com/AcalephStorage/kontinuous/kube" @@ -67,7 +70,6 @@ func (p *PipelineResource) Register(container *restful.Container) { Operation("definition"). Param(ws.PathParameter("owner", "repository owner name").DataType("string")). Param(ws.PathParameter("repo", "repository name").DataType("string")). - Writes(ps.Definition{}). Filter(requireAccessToken)) ws.Route(ws.GET("/{owner}/{repo}/definition/{ref}").To(p.definition). @@ -76,7 +78,15 @@ func (p *PipelineResource) Register(container *restful.Container) { Param(ws.PathParameter("owner", "repository owner name").DataType("string")). Param(ws.PathParameter("repo", "repository name").DataType("string")). Param(ws.PathParameter("ref", "commit or branch").DataType("string")). - Writes(ps.Definition{}). + Filter(requireAccessToken)) + + ws.Route(ws.POST("/{owner}/{repo}/definition/{commit}").To(p.updateDefinition). + Doc("Update definition file of the pipeline"). + Operation("updateDefinition"). + Param(ws.PathParameter("owner", "repository owner name").DataType("string")). + Param(ws.PathParameter("repo", "repository name").DataType("string")). + Param(ws.PathParameter("commit", "commit ref").DataType("string")). + Consumes("text/plain"). Filter(requireAccessToken)) buildResource := &BuildResource{ @@ -174,11 +184,39 @@ func (p *PipelineResource) definition(req *restful.Request, res *restful.Respons return } - definition, err := pipeline.Definition(ref, client) - if err != nil { + file, exists := client.GetContents(pipeline.Owner, pipeline.Repo, ps.PipelineYAML, ref) + if !exists { jsonError(res, http.StatusNotFound, err, fmt.Sprintf("Unable to fetch definition for %s/%s", owner, repo)) return } - res.WriteEntity(definition) + res.WriteAsJson(file) +} + +func (p *PipelineResource) updateDefinition(req *restful.Request, res *restful.Response) { + client := newSCMClient(req) + owner := req.PathParameter("owner") + repo := req.PathParameter("repo") + commit := req.PathParameter("commit") + + pipeline, err := findPipeline(owner, repo, p.KVClient) + if err != nil { + jsonError(res, http.StatusNotFound, err, fmt.Sprintf("Unable to find pipeline %s/%s", owner, repo)) + return + } + + body, _ := ioutil.ReadAll(req.Request.Body) + content, err := base64.URLEncoding.DecodeString(string(body)) + if err != nil { + jsonError(res, http.StatusInternalServerError, err, fmt.Sprintf("Unable to decode %s", string(body))) + return + } + + err = client.UpdateFile(pipeline.Owner, pipeline.Repo, ps.PipelineYAML, commit, content) + if err != nil { + jsonError(res, http.StatusInternalServerError, err, fmt.Sprintf("Unable to update file %s", ps.PipelineYAML)) + return + } + + res.WriteHeader(http.StatusAccepted) } diff --git a/pipeline/common_test.go b/pipeline/common_test.go index d157c0f..30c9616 100644 --- a/pipeline/common_test.go +++ b/pipeline/common_test.go @@ -230,7 +230,13 @@ func (s MockSCMClient) GetContents(owner, repo, path, ref string) (*scm.Reposito if !s.success { return nil, false } - return &scm.RepositoryContent{validyamlSpec}, true + return &scm.RepositoryContent{ + Content: &validyamlSpec, + }, true +} + +func (s MockSCMClient) UpdateFile(owner, repo, path, commit string, content []byte) error { + return nil } func (s MockSCMClient) ListRepositories(user string) ([]*scm.Repository, error) { @@ -241,6 +247,6 @@ func (s MockSCMClient) ParseHook(payload []byte, event string) (*scm.Hook, error return nil, nil } -func (s MockSCMClient) CreateStatus(owner, repo, sha string, stageId int, stageName, state string) error { +func (s MockSCMClient) CreateStatus(owner, repo, sha string, stageID int, stageName, state string) error { return nil } diff --git a/pipeline/definition_test.go b/pipeline/definition_test.go index 3cd1ff6..5fff1f1 100644 --- a/pipeline/definition_test.go +++ b/pipeline/definition_test.go @@ -4,55 +4,8 @@ import ( "testing" ) -var validyamlSpec string - func TestReadValidPipeline(t *testing.T) { - validyamlSpec = ` -apiVersion: v1alpha1 -kind: Pipeline -metadata: - name: my-pipeline - namespace: acaleph -spec: - selector: - matchLabels: - app: my-pipeline-infra # can be used as a selector for finding infra launched during a build? - template: # taken from Job spec... needed? - metadata: - name: my-pipeline - labels: - app: my-pipeline - image: acaleph/deploy-base # Overridable? - stages: - - name: Test Infra - type: command - params: - command: ["test_infra.sh", "--hack-the-gibson"] - env: # env vars - - name: MYVAR - value: my value - artifact_paths: # sent to minio? - - "logs/**/*" - - "coverage/**/*" - timeout: 60 # kills after X minutes? - - name: "Have you finished testing?" # friendly name - type: block # waits for the user to approve - - name: Teardown Deploy local test infra - type: deploy_cleanup # stop - selector: - build: my-val-stage-1 # selects build to stop - - name: "Do you want to Publish to Production?" # friendly name - type: block # waits for the user to approve - - name: Deploy production - type: deploy - params: - template: production.yaml # DM template - properties: # DM properties - external_service: true - replicas: 3 - labels: # labels to cleanup build - state: canary - ` + // validyamlSpec in common_test.go _, err := GetDefinition([]byte(validyamlSpec)) if err == nil { diff --git a/scm/client.go b/scm/client.go index 99cd834..42318fc 100644 --- a/scm/client.go +++ b/scm/client.go @@ -4,7 +4,7 @@ const ( // EventDashboard indicates a dashboard event EventDashboard = "dashboard" - // EventDashboard indicates a CLI event + // EventCLI indicates a CLI event EventCLI = "cli" // EventPing indicates a ping event @@ -43,9 +43,10 @@ type Client interface { HookExists(owner, repo, url string) bool CreateHook(owner, repo, callback string, events []string) error CreateKey(owner, repo, key, title string) error - CreateStatus(owner, repo, sha string, stageId int, stageName, state string) error + CreateStatus(owner, repo, sha string, stageID int, stageName, state string) error GetFileContent(owner, repo, path, ref string) ([]byte, bool) GetContents(owner, repo, path, ref string) (*RepositoryContent, bool) + UpdateFile(owner, repo, path, commit string, content []byte) error GetRepository(owner, repo string) (*Repository, bool) ListRepositories(user string) ([]*Repository, error) ParseHook(payload []byte, event string) (*Hook, error) @@ -63,6 +64,7 @@ type Repository struct { Permissions map[string]bool `json:"-"` } +// RepositoryContent contains metadata of a file/directory in a repository type RepositoryContent struct { Content *string `json:"content"` SHA *string `json:"sha,omitempty"` diff --git a/scm/github/github.go b/scm/github/github.go index 5588526..0ac410b 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -57,9 +57,9 @@ func (gc *Client) CreateKey(owner, repo, key, title string) error { return nil } -func (gc *Client) CreateStatus(owner, repo, ref string, stageId int, stageName, state string) error { +func (gc *Client) CreateStatus(owner, repo, ref string, stageID int, stageName, state string) error { - context := fmt.Sprintf("kontinuous:%d", stageId) + context := fmt.Sprintf("kontinuous:%d", stageID) status := &github.RepoStatus{ State: &state, @@ -102,17 +102,29 @@ func (gc *Client) GetContents(owner, repo, path, ref string) (*scm.RepositoryCon return nil, false } - decoded, err := file.GetContent() - if err != nil { - return nil, false - } - return &scm.RepositoryContent{ - Content: &decoded, + Content: file.Content, SHA: file.SHA, }, true } +// UpdateFile commits diff of a file content from a given commit ref +func (gc *Client) UpdateFile(owner, repo, path, commit string, content []byte) error { + message := fmt.Sprintf("Update %s", path) + _, _, err := gc.client().Repositories.UpdateFile(owner, + repo, + path, + &github.RepositoryContentFileOptions{ + Message: &message, + Content: content, + SHA: &commit, + }) + if err != nil { + return err + } + return nil +} + // GetRepository fetches repository details from GitHub func (gc *Client) GetRepository(owner, name string) (*scm.Repository, bool) { data, _, err := gc.client().Repositories.Get(owner, name) From 637a16586e3f9e7ed0156acb04e0a92a75dc8830 Mon Sep 17 00:00:00 2001 From: Stefany Serino Date: Tue, 17 May 2016 16:42:03 +0800 Subject: [PATCH 10/30] Return updated commit reference of pipeline definition --- api/pipeline.go | 15 ++++++++++----- pipeline/common_test.go | 4 ++-- scm/client.go | 6 +++--- scm/github/github.go | 11 +++++++---- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/api/pipeline.go b/api/pipeline.go index 2947932..3bd8adc 100644 --- a/api/pipeline.go +++ b/api/pipeline.go @@ -4,6 +4,7 @@ import ( "fmt" "encoding/base64" + "encoding/json" "io/ioutil" "net/http" @@ -86,7 +87,6 @@ func (p *PipelineResource) Register(container *restful.Container) { Param(ws.PathParameter("owner", "repository owner name").DataType("string")). Param(ws.PathParameter("repo", "repository name").DataType("string")). Param(ws.PathParameter("commit", "commit ref").DataType("string")). - Consumes("text/plain"). Filter(requireAccessToken)) buildResource := &BuildResource{ @@ -206,17 +206,22 @@ func (p *PipelineResource) updateDefinition(req *restful.Request, res *restful.R } body, _ := ioutil.ReadAll(req.Request.Body) - content, err := base64.URLEncoding.DecodeString(string(body)) + var payload map[string]string + if err := json.Unmarshal(body, &payload); err != nil { + jsonError(res, http.StatusInternalServerError, err, "Unable to read request payload") + return + } + decodedContent, err := base64.URLEncoding.DecodeString(payload["content"]) if err != nil { - jsonError(res, http.StatusInternalServerError, err, fmt.Sprintf("Unable to decode %s", string(body))) + jsonError(res, http.StatusInternalServerError, err, fmt.Sprintf("Unable to decode %s", payload["content"])) return } - err = client.UpdateFile(pipeline.Owner, pipeline.Repo, ps.PipelineYAML, commit, content) + file, err := client.UpdateFile(pipeline.Owner, pipeline.Repo, ps.PipelineYAML, commit, decodedContent) if err != nil { jsonError(res, http.StatusInternalServerError, err, fmt.Sprintf("Unable to update file %s", ps.PipelineYAML)) return } - res.WriteHeader(http.StatusAccepted) + res.WriteAsJson(file) } diff --git a/pipeline/common_test.go b/pipeline/common_test.go index 30c9616..19fd309 100644 --- a/pipeline/common_test.go +++ b/pipeline/common_test.go @@ -235,8 +235,8 @@ func (s MockSCMClient) GetContents(owner, repo, path, ref string) (*scm.Reposito }, true } -func (s MockSCMClient) UpdateFile(owner, repo, path, commit string, content []byte) error { - return nil +func (s MockSCMClient) UpdateFile(owner, repo, path, commit string, content []byte) (*scm.RepositoryContent, error) { + return &scm.RepositoryContent{}, nil } func (s MockSCMClient) ListRepositories(user string) ([]*scm.Repository, error) { diff --git a/scm/client.go b/scm/client.go index 42318fc..eccdcc3 100644 --- a/scm/client.go +++ b/scm/client.go @@ -46,7 +46,7 @@ type Client interface { CreateStatus(owner, repo, sha string, stageID int, stageName, state string) error GetFileContent(owner, repo, path, ref string) ([]byte, bool) GetContents(owner, repo, path, ref string) (*RepositoryContent, bool) - UpdateFile(owner, repo, path, commit string, content []byte) error + UpdateFile(owner, repo, path, commit string, content []byte) (*RepositoryContent, error) GetRepository(owner, repo string) (*Repository, bool) ListRepositories(user string) ([]*Repository, error) ParseHook(payload []byte, event string) (*Hook, error) @@ -66,8 +66,8 @@ type Repository struct { // RepositoryContent contains metadata of a file/directory in a repository type RepositoryContent struct { - Content *string `json:"content"` - SHA *string `json:"sha,omitempty"` + Content *string `json:"content,omitempty"` + SHA *string `json:"sha"` } // IsAdmin determines if the scoped user has admin rights for the repository diff --git a/scm/github/github.go b/scm/github/github.go index 0ac410b..4cc2ec8 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -109,9 +109,9 @@ func (gc *Client) GetContents(owner, repo, path, ref string) (*scm.RepositoryCon } // UpdateFile commits diff of a file content from a given commit ref -func (gc *Client) UpdateFile(owner, repo, path, commit string, content []byte) error { +func (gc *Client) UpdateFile(owner, repo, path, commit string, content []byte) (*scm.RepositoryContent, error) { message := fmt.Sprintf("Update %s", path) - _, _, err := gc.client().Repositories.UpdateFile(owner, + resp, _, err := gc.client().Repositories.UpdateFile(owner, repo, path, &github.RepositoryContentFileOptions{ @@ -120,9 +120,12 @@ func (gc *Client) UpdateFile(owner, repo, path, commit string, content []byte) e SHA: &commit, }) if err != nil { - return err + return nil, err } - return nil + + return &scm.RepositoryContent{ + SHA: resp.Content.SHA, + }, nil } // GetRepository fetches repository details from GitHub From 0fada16364e5fb897b297cb62b30b4606c58ce7d Mon Sep 17 00:00:00 2001 From: dexter Date: Wed, 18 May 2016 08:51:28 +0800 Subject: [PATCH 11/30] moved sensitive data to secrets and cli deploy fixes --- README.md | 6 +++++- cli/cli.go | 9 +++++++-- cli/deploy.go | 9 +++++++-- cmd/kontinuous.go | 10 +++++++--- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 71953e6..fb35773 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,9 @@ A Kubernetes Secret also needs to be defined and mounted to the Pod. The secret { "AuthSecret": "base64 encoded auth secret", "S3SecretKey": "s3 secret key", - "S3AccessKey": "s3 access key" + "S3AccessKey": "s3 access key", + "GithubClientID": "github client ID", + "GithubClientSecret": "github client secret" } ``` @@ -92,6 +94,8 @@ A Kubernetes Secret also needs to be defined and mounted to the Pod. The secret `S3SecretKey` and `S3AccessKey` are the keys needed to access minio (or S3). +`GithubClientID` and `GithubClientSecret` are optional and only required for Github only authentication. (See below) + The secret needs to be mounted to the Pod to the path `/.secret`. ## Using Kontinuous diff --git a/cli/cli.go b/cli/cli.go index f1d5aff..98cf7ab 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -150,6 +150,10 @@ func main() { Name: "clientid", Usage: "Required, github client id", }, + cli.StringFlag{ + Name: "clientsecret", + Usage: "Required, github client secret", + }, }, Action: deployApp, @@ -363,16 +367,17 @@ func deployApp(c *cli.Context) { secretKey := c.String("secretkey") authCode := c.String("authcode") clientId := c.String("clientid") + clientSecret := c.String("clientsecret") missingFields := false - if namespace == "" || accessKey == "" || secretKey == "" || authCode == "" || clientId == "" { + if namespace == "" || accessKey == "" || secretKey == "" || authCode == "" || clientId == "" || clientSecret == "" { fmt.Println("missing required fields!") missingFields = true } if !missingFields { - err := DeployKontinuous(namespace, accessKey, secretKey, authCode, clientId) + err := DeployKontinuous(namespace, accessKey, secretKey, authCode, clientId, clientSecret) if err != nil { fmt.Println("Oops something went wrong. Unable to deploy kontinuous.") fmt.Println(err) diff --git a/cli/deploy.go b/cli/deploy.go index 9f91e59..a124a5b 100644 --- a/cli/deploy.go +++ b/cli/deploy.go @@ -14,7 +14,9 @@ var secretData = ` { "AuthSecret": "{{.AuthCode}}", "S3SecretKey": "{{.SecretKey}}", - "S3AccessKey": "{{.AccessKey}}" + "S3AccessKey": "{{.AccessKey}}", + "GithubClientID": "{{.GHClient}}", + "GithubClientSecret": "{{.GHSecret}}" } ` @@ -331,6 +333,7 @@ type Deploy struct { KontinuousIP string DashboardIP string GHClient string + GHSecret string } const ( @@ -449,13 +452,15 @@ func RemoveResources() error { return nil } -func DeployKontinuous(namespace, accesskey, secretkey, authcode, clientid string) error { +func DeployKontinuous(namespace, accesskey, secretkey, authcode, clientid, clientsecret string) error { + fmt.Println("Deploying Kontinuous...") deploy := Deploy{ Namespace: namespace, AccessKey: accesskey, SecretKey: secretkey, AuthCode: authcode, GHClient: clientid, + GHSecret: clientsecret, } sData, _ := generateResource(secretData, &deploy) encryptedSecret, _ := encryptSecret(sData) diff --git a/cmd/kontinuous.go b/cmd/kontinuous.go index 0fe74bc..7b0b987 100644 --- a/cmd/kontinuous.go +++ b/cmd/kontinuous.go @@ -25,9 +25,11 @@ const ( ) type Secrets struct { - AuthSecret string - S3SecretKey string - S3AccessKey string + AuthSecret string + S3SecretKey string + S3AccessKey string + GithubClientID string + GithubClientSecret string } var ( @@ -174,5 +176,7 @@ func setEnv() { os.Setenv("AUTH_SECRET", secrets.AuthSecret) os.Setenv("S3_ACCESS_KEY", secrets.S3AccessKey) os.Setenv("S3_SECRET_KEY", secrets.S3SecretKey) + os.Setenv("GITHUB_CLIENT_ID", secrets.GithubClientID) + os.Setenv("GITHUB_CLIENT_SECRET", secrets.GithubClientSecret) } } From 0a4a90d458ffa6a03f34781733fc543408893cdd Mon Sep 17 00:00:00 2001 From: dexter Date: Wed, 18 May 2016 10:20:45 +0800 Subject: [PATCH 12/30] GH -> GITHUB. --- api/auth.go | 4 ++-- cli/deploy.go | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/api/auth.go b/api/auth.go index cc25b7a..dc43d1a 100644 --- a/api/auth.go +++ b/api/auth.go @@ -115,8 +115,8 @@ func (a *AuthResource) githubLogin(req *restful.Request, res *restful.Response) Path: "login/oauth/access_token", } q := reqUrl.Query() - q.Set("client_id", os.Getenv("GH_CLIENT_ID")) - q.Set("client_secret", os.Getenv("GH_CLIENT_SECRET")) + q.Set("client_id", os.Getenv("GITHUB_CLIENT_ID")) + q.Set("client_secret", os.Getenv("GITHUB_CLIENT_SECRET")) q.Set("code", authCode) q.Set("state", state) reqUrl.RawQuery = q.Encode() diff --git a/cli/deploy.go b/cli/deploy.go index a124a5b..c3192ca 100644 --- a/cli/deploy.go +++ b/cli/deploy.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/base64" "fmt" "os" "os/exec" @@ -60,6 +61,9 @@ spec: app: minio type: object-store spec: + volumes: + - name: empty-dir + emptyDir: {} containers: - name: minio image: minio/minio:latest @@ -69,6 +73,11 @@ spec: value: {{.AccessKey}} - name: MINIO_SECRET_KEY value: {{.SecretKey}} + args: + - /data + volumeMounts: + - name: empty-dir + mountPath: /data ports: - name: service containerPort: 9000 @@ -391,14 +400,8 @@ func saveToFile(path string, data ...string) error { return nil } -func encryptSecret(secret string) (string, error) { - cmd := fmt.Sprintf(`echo -n "%s" | openssl base64 | tr -d "\n"`, secret) - out, err := exec.Command("bash", "-c", cmd).Output() - if err != nil { - return "", err - } - return string(out), nil - +func encryptSecret(secret string) string { + return base64.StdEncoding.EncodeToString([]byte(secret)) } func createKontinuousResouces(path string) error { @@ -463,8 +466,7 @@ func DeployKontinuous(namespace, accesskey, secretkey, authcode, clientid, clien GHSecret: clientsecret, } sData, _ := generateResource(secretData, &deploy) - encryptedSecret, _ := encryptSecret(sData) - deploy.SecretData = encryptedSecret + deploy.SecretData = encryptSecret(sData) secret, _ := generateResource(secret, &deploy) minioStr, _ := generateResource(minio, &deploy) etcdStr, _ := generateResource(etcd, &deploy) From b4c8424efd7bd8485a3ca93db02edc1bfc85e6d5 Mon Sep 17 00:00:00 2001 From: dexter Date: Wed, 18 May 2016 15:40:19 +0800 Subject: [PATCH 13/30] detailed flags and default namespace --- cli/cli.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index 98cf7ab..9973884 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -132,27 +132,28 @@ func main() { Flags: []cli.Flag{ cli.StringFlag{ Name: "namespace", - Usage: "Required, kontinuous namespace", + Usage: "Required, Kubernetes namespace to deploy kontinuous", + Value: "kontinuous", }, cli.StringFlag{ - Name: "accesskey", - Usage: "Required, s3 access key", + Name: "s3-access-key", + Usage: "Required, S3 access key", }, cli.StringFlag{ - Name: "secretkey", - Usage: "Required, s3 secret key", + Name: "s3-secret-key", + Usage: "Required, S3 secret key", }, cli.StringFlag{ - Name: "authcode", - Usage: "Required, jwt authorization code", + Name: "auth-secret", + Usage: "Required, base64 encoded secret to sign JWT", }, cli.StringFlag{ - Name: "clientid", - Usage: "Required, github client id", + Name: "github-client-id", + Usage: "Required, Github Client ID for github authentication", }, cli.StringFlag{ - Name: "clientsecret", - Usage: "Required, github client secret", + Name: "github-client-secret", + Usage: "Required, Github Client Secret for github authentication", }, }, @@ -363,11 +364,11 @@ func deleteBuild(c *cli.Context) { func deployApp(c *cli.Context) { namespace := c.String("namespace") - accessKey := c.String("accesskey") - secretKey := c.String("secretkey") - authCode := c.String("authcode") - clientId := c.String("clientid") - clientSecret := c.String("clientsecret") + accessKey := c.String("s3-access-key") + secretKey := c.String("s3-secret-key") + authCode := c.String("auth-secret") + clientId := c.String("github-client-id") + clientSecret := c.String("github-client-secret") missingFields := false if namespace == "" || accessKey == "" || secretKey == "" || authCode == "" || clientId == "" || clientSecret == "" { From ada7e4c9876406cfed9ea57d0919d0b01c857288 Mon Sep 17 00:00:00 2001 From: dexter Date: Wed, 18 May 2016 18:15:27 +0800 Subject: [PATCH 14/30] added temp refactor of README --- README.md | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bb1081a..cb6cdac 100644 --- a/README.md +++ b/README.md @@ -26,13 +26,35 @@ We've got lots more planned, see the [Roadmap](#roadmap) or Github issues to get ### Getting Started -The script `scripts/kontinuous-deploy` is a quick way of running `kontinuous` in a K8s cluster. The general syntax is: +Before running Kontinuous, it needs to be added as a github OAuth Application [here](https://github.com/settings/applications/new). The `Client ID` and `Client Secret` will be used in running Kontinuous. + +The `kubernetes-cli` can bootstrap a kontinuous setup on a running Kubernetes cluster. This requires `kubectl` to be in the `PATH` and configured to access the cluster. ``` -$ kontinuous-deploy --namespace {k8s-namespace} --auth-secret {base64url encoded secret} --s3-access-key {s3 access key} --s3-secret-key {s3 secret key} +$ kotinuous-cli --namespace {namespace} \ + --auth-secret {base64 encoded secret} \ + --github-client-id {github client id} \ + --github-client-secret {github client secret} ``` -This will launch `kontinuous` via the locally configured `kubectl` in the given namespace together with `etcd`, `minio`, and a docker `registry`. This expects that the kubernetes cluster supports the LoadBalancer service. +Parameters: + +| parameter | description | +|------------------------|------------------------------------------------------------------------------------------------------------------------------| +| --namespace | The namespace to deploy Kontinuous to. This defaults to `kontinuous` | +| --auth-secret | A base64 encoded secret. This is used by kontinuous to provide JWT for authentication. This can be any base64 encoded string | +| --github-client-id | The Github client ID provided when registering kontinuous as a Github OAuth application | +| --github-client-secret | The Github client secret provided when registering kontinuous as a Github OAuth application | + +This will launch the following via `kubectl`: + + - etcd + - minio + - docker registry + - kontinuous + - kontinuous-ui + +This will launch `kontinuous` via the locally configured `kubectl` in the given namespace together with `etcd`, `minio`, a docker `registry`, and the Kontinuous UI. This expects that the kubernetes cluster supports the LoadBalancer service. Alternatively, for more customization, a sample yaml file for running kontinuous and its dependencies in Kubernetes can be found [here](./k8s-spec.yml.example). See [below](#running-in-kubernetes) for how to configure secrets. @@ -228,6 +250,9 @@ Optional params are: `deploy` deploys a Kubernetes Spec file (yaml) to kubernetes. +--- +# TODO: WIP. This section is about accessing the API. not necessary if using the UI. Might move this to a separate section...(?) +--- ### Authentication From 00c9eea505ee9d3a60ceccebc9fe6047a87e58c4 Mon Sep 17 00:00:00 2001 From: Stefany Serino Date: Wed, 18 May 2016 22:05:59 +0800 Subject: [PATCH 15/30] Add pull_request option for updating definition file --- api/pipeline.go | 29 +++++++++--------- pipeline/common_test.go | 12 +++++++- pipeline/definition.go | 6 ++++ pipeline/pipeline.go | 67 +++++++++++++++++++++++++++++++++++++++++ scm/client.go | 5 ++- scm/github/github.go | 65 ++++++++++++++++++++++++++++++++++----- 6 files changed, 159 insertions(+), 25 deletions(-) diff --git a/api/pipeline.go b/api/pipeline.go index 3bd8adc..ce8f21a 100644 --- a/api/pipeline.go +++ b/api/pipeline.go @@ -3,7 +3,6 @@ package api import ( "fmt" - "encoding/base64" "encoding/json" "io/ioutil" "net/http" @@ -71,6 +70,7 @@ func (p *PipelineResource) Register(container *restful.Container) { Operation("definition"). Param(ws.PathParameter("owner", "repository owner name").DataType("string")). Param(ws.PathParameter("repo", "repository name").DataType("string")). + Writes(ps.DefinitionFile{}). Filter(requireAccessToken)) ws.Route(ws.GET("/{owner}/{repo}/definition/{ref}").To(p.definition). @@ -79,14 +79,15 @@ func (p *PipelineResource) Register(container *restful.Container) { Param(ws.PathParameter("owner", "repository owner name").DataType("string")). Param(ws.PathParameter("repo", "repository name").DataType("string")). Param(ws.PathParameter("ref", "commit or branch").DataType("string")). + Writes(ps.DefinitionFile{}). Filter(requireAccessToken)) - ws.Route(ws.POST("/{owner}/{repo}/definition/{commit}").To(p.updateDefinition). + ws.Route(ws.PUT("/{owner}/{repo}/definition").To(p.updateDefinition). Doc("Update definition file of the pipeline"). Operation("updateDefinition"). Param(ws.PathParameter("owner", "repository owner name").DataType("string")). Param(ws.PathParameter("repo", "repository name").DataType("string")). - Param(ws.PathParameter("commit", "commit ref").DataType("string")). + Writes(ps.DefinitionFile{}). Filter(requireAccessToken)) buildResource := &BuildResource{ @@ -184,8 +185,9 @@ func (p *PipelineResource) definition(req *restful.Request, res *restful.Respons return } - file, exists := client.GetContents(pipeline.Owner, pipeline.Repo, ps.PipelineYAML, ref) + file, exists := pipeline.GetDefinitionFile(client, ref) if !exists { + err = fmt.Errorf("Definition file for %s/%s not found.", owner, repo) jsonError(res, http.StatusNotFound, err, fmt.Sprintf("Unable to fetch definition for %s/%s", owner, repo)) return } @@ -194,10 +196,8 @@ func (p *PipelineResource) definition(req *restful.Request, res *restful.Respons } func (p *PipelineResource) updateDefinition(req *restful.Request, res *restful.Response) { - client := newSCMClient(req) owner := req.PathParameter("owner") repo := req.PathParameter("repo") - commit := req.PathParameter("commit") pipeline, err := findPipeline(owner, repo, p.KVClient) if err != nil { @@ -205,23 +205,22 @@ func (p *PipelineResource) updateDefinition(req *restful.Request, res *restful.R return } + client := newSCMClient(req) body, _ := ioutil.ReadAll(req.Request.Body) - var payload map[string]string + type definitionPayload struct { + Definition *ps.DefinitionFile `json:"definition"` + Commit map[string]string `json:"commit"` + } + payload := &definitionPayload{} if err := json.Unmarshal(body, &payload); err != nil { jsonError(res, http.StatusInternalServerError, err, "Unable to read request payload") return } - decodedContent, err := base64.URLEncoding.DecodeString(payload["content"]) - if err != nil { - jsonError(res, http.StatusInternalServerError, err, fmt.Sprintf("Unable to decode %s", payload["content"])) - return - } - file, err := client.UpdateFile(pipeline.Owner, pipeline.Repo, ps.PipelineYAML, commit, decodedContent) + file, err := pipeline.UpdateDefinitionFile(client, payload.Definition, payload.Commit) if err != nil { - jsonError(res, http.StatusInternalServerError, err, fmt.Sprintf("Unable to update file %s", ps.PipelineYAML)) + jsonError(res, http.StatusInternalServerError, err, fmt.Sprintf("Unable to update definition file for %s/%s", pipeline.Owner, pipeline.Repo)) return } - res.WriteAsJson(file) } diff --git a/pipeline/common_test.go b/pipeline/common_test.go index 19fd309..32a9e09 100644 --- a/pipeline/common_test.go +++ b/pipeline/common_test.go @@ -235,7 +235,7 @@ func (s MockSCMClient) GetContents(owner, repo, path, ref string) (*scm.Reposito }, true } -func (s MockSCMClient) UpdateFile(owner, repo, path, commit string, content []byte) (*scm.RepositoryContent, error) { +func (s MockSCMClient) UpdateFile(owner, repo, path, blob, message, branch string, content []byte) (*scm.RepositoryContent, error) { return &scm.RepositoryContent{}, nil } @@ -250,3 +250,13 @@ func (s MockSCMClient) ParseHook(payload []byte, event string) (*scm.Hook, error func (s MockSCMClient) CreateStatus(owner, repo, sha string, stageID int, stageName, state string) error { return nil } + +func (s MockSCMClient) GetHead(owner, repo, branch string) (string, error) { + return "", nil +} +func (s MockSCMClient) CreateBranch(owner, repo, branchName, baseRef string) (string, error) { + return "", nil +} +func (s MockSCMClient) CreatePullRequest(owner, repo, baseRef, headRef, title string) error { + return nil +} diff --git a/pipeline/definition.go b/pipeline/definition.go index a996eee..c87f57e 100644 --- a/pipeline/definition.go +++ b/pipeline/definition.go @@ -31,6 +31,12 @@ type Definition struct { Spec SpecDetails `json:"spec"` } +// DefinitionFile holds repository metadata of the definition file (PipelineYAML) +type DefinitionFile struct { + Content *string `json:"content,omitempty"` + SHA *string `json:"sha"` +} + func (d *Definition) GetStages() []*Stage { stages := make([]*Stage, len(d.Spec.Template.Stages)) diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index 640b98c..4eeb0ca 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -520,3 +520,70 @@ func (p *Pipeline) UpdatePipeline(definition *Definition, kvClient kv.KVClient) p.Save(kvClient) } + +// GetDefinitionFile fetches the definition file (PipelineYAML) from the pipeline's repository +// returns the content (possibly encoded in base64, see scm API) and +// the SHA of the file (blob) +func (p *Pipeline) GetDefinitionFile(c scm.Client, ref string) (*DefinitionFile, bool) { + file, exists := c.GetContents(p.Owner, p.Repo, PipelineYAML, ref) + if !exists { + return nil, false + } + return &DefinitionFile{ + Content: file.Content, + SHA: file.SHA, + }, true +} + +// UpdateDefinitionFile commits the changes of the definition file (PipelineYAML) +// either directly to the default branch +// or through a pull request +func (p *Pipeline) UpdateDefinitionFile(c scm.Client, file *DefinitionFile, commit map[string]string) (*DefinitionFile, error) { + source, exists := c.GetRepository(p.Owner, p.Repo) + if !exists { + return nil, fmt.Errorf("Unable to find repository %s/%s", p.Owner, p.Repo) + } + defaultBranch := source.DefaultBranch + branch := defaultBranch + + message := commit["message"] + if len(message) == 0 { + message = fmt.Sprintf("Update %s", PipelineYAML) + } + + if commit["option"] == "pull_request" { + branch = commit["branch_name"] + if len(branch) == 0 { + return nil, fmt.Errorf("Branch name not provided") + } + + head, err := c.GetHead(p.Owner, p.Repo, defaultBranch) + if err != nil { + return nil, err + } + _, err = c.CreateBranch(p.Owner, p.Repo, branch, head) + if err != nil { + return nil, err + } + // wait for branch to be created (todo: verify existence of branch) + time.Sleep(3 * time.Second) + } + + decodedContent, err := base64.URLEncoding.DecodeString(*file.Content) + if err != nil { + return nil, err + } + updatedFile, err := c.UpdateFile(p.Owner, p.Repo, PipelineYAML, *file.SHA, message, branch, decodedContent) + if err != nil { + return nil, err + } + + if commit["option"] == "pull_request" { + err = c.CreatePullRequest(p.Owner, p.Repo, defaultBranch, branch, message) + if err != nil { + return nil, err + } + } + + return &DefinitionFile{SHA: updatedFile.SHA}, nil +} diff --git a/scm/client.go b/scm/client.go index eccdcc3..833d739 100644 --- a/scm/client.go +++ b/scm/client.go @@ -46,10 +46,13 @@ type Client interface { CreateStatus(owner, repo, sha string, stageID int, stageName, state string) error GetFileContent(owner, repo, path, ref string) ([]byte, bool) GetContents(owner, repo, path, ref string) (*RepositoryContent, bool) - UpdateFile(owner, repo, path, commit string, content []byte) (*RepositoryContent, error) + UpdateFile(owner, repo, path, blob, message, branch string, content []byte) (*RepositoryContent, error) GetRepository(owner, repo string) (*Repository, bool) ListRepositories(user string) ([]*Repository, error) ParseHook(payload []byte, event string) (*Hook, error) + GetHead(owner, repo, branch string) (string, error) + CreateBranch(owner, repo, branchName, baseRef string) (string, error) + CreatePullRequest(owner, repo, baseRef, headRef, title string) error } // Repository holds common repository details from SCMs diff --git a/scm/github/github.go b/scm/github/github.go index 4cc2ec8..eb256e8 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -6,6 +6,7 @@ import ( "encoding/json" + "github.com/Sirupsen/logrus" "github.com/google/go-github/github" "golang.org/x/oauth2" @@ -108,24 +109,24 @@ func (gc *Client) GetContents(owner, repo, path, ref string) (*scm.RepositoryCon }, true } -// UpdateFile commits diff of a file content from a given commit ref -func (gc *Client) UpdateFile(owner, repo, path, commit string, content []byte) (*scm.RepositoryContent, error) { - message := fmt.Sprintf("Update %s", path) +// UpdateFile commits diff of a file content from the given blob +func (gc *Client) UpdateFile(owner, repo, path, blob, message, branch string, content []byte) (*scm.RepositoryContent, error) { + if len(message) == 0 { + message = fmt.Sprintf("Update %s", path) + } resp, _, err := gc.client().Repositories.UpdateFile(owner, repo, path, &github.RepositoryContentFileOptions{ Message: &message, Content: content, - SHA: &commit, + SHA: &blob, + Branch: &branch, }) if err != nil { return nil, err } - - return &scm.RepositoryContent{ - SHA: resp.Content.SHA, - }, nil + return &scm.RepositoryContent{SHA: resp.Content.SHA}, nil } // GetRepository fetches repository details from GitHub @@ -228,3 +229,51 @@ func (gc *Client) SetAccessToken(token string) { func (gc *Client) Name() string { return scm.RepoGithub } + +// GetHead gets the HEAD ref of a branch +func (gc *Client) GetHead(owner, repo, branch string) (string, error) { + // func (s *GitService) GetRef(owner string, repo string, ref string) (*Reference, *Response, error) + ref := fmt.Sprintf("refs/heads/%s", branch) + reference, _, err := gc.client().Git.GetRef(owner, repo, ref) + if err != nil { + return "", err + } + + return *reference.Object.SHA, nil +} + +// CreateBranch creates a new branch of the repository from a commit as baseRef +func (gc *Client) CreateBranch(owner, repo, branchName, baseRef string) (string, error) { + headRef := fmt.Sprintf("refs/heads/%s", branchName) + reference, _, err := gc.client().Git.CreateRef( + owner, + repo, + &github.Reference{ + Ref: &headRef, + Object: &github.GitObject{SHA: &baseRef}, + }, + ) + if err != nil { + return "", err + } + return *reference.Ref, nil +} + +// CreatePullRequest starts a pull request of the changes from headRef to baseRef +func (gc *Client) CreatePullRequest(owner, repo, baseRef, headRef, title string) error { + // func (s *PullRequestsService) Create(owner string, repo string, pull *NewPullRequest) (*PullRequest, *Response, error) + _, _, err := gc.client().PullRequests.Create( + owner, + repo, + &github.NewPullRequest{ + Title: &title, + Head: &headRef, + Base: &baseRef, + }, + ) + if err != nil { + logrus.WithError(err).Error("Error creating pull request") + return err + } + return nil +} From 5eb7d7a1fadd28a271f964289fe8b354ae9c0f2a Mon Sep 17 00:00:00 2001 From: madz Date: Thu, 19 May 2016 10:13:14 +0800 Subject: [PATCH 16/30] add init command. removed s3 secret and access key flags. individual deployment for resources --- cli/cli.go | 77 ++++++++++++++------ cli/deploy.go | 197 +++++++++++++++++++++++++++++++++++--------------- cli/init.go | 106 +++++++++++++++++++++++++++ 3 files changed, 302 insertions(+), 78 deletions(-) create mode 100644 cli/init.go diff --git a/cli/cli.go b/cli/cli.go index 9973884..50329f4 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -125,40 +125,53 @@ func main() { Subcommands: []cli.Command{ { Name: "remove", - Usage: "remove kontinuous app in the cluster", + Usage: "remove Kontinuous resources in the cluster", Action: removeDeployedApp, }, }, Flags: []cli.Flag{ cli.StringFlag{ Name: "namespace", - Usage: "Required, Kubernetes namespace to deploy kontinuous", + Usage: "Required, Kubernetes namespace to deploy Kontinuous", Value: "kontinuous", }, - cli.StringFlag{ - Name: "s3-access-key", - Usage: "Required, S3 access key", - }, - cli.StringFlag{ - Name: "s3-secret-key", - Usage: "Required, S3 secret key", - }, cli.StringFlag{ Name: "auth-secret", Usage: "Required, base64 encoded secret to sign JWT", }, cli.StringFlag{ Name: "github-client-id", - Usage: "Required, Github Client ID for github authentication", + Usage: "Required, Github Client ID for Github authentication", }, cli.StringFlag{ Name: "github-client-secret", - Usage: "Required, Github Client Secret for github authentication", + Usage: "Required, Github Client Secret for Github authentication", }, }, Action: deployApp, }, + { + Name: "init", + Usage: "create and save initial .pipeline.yml in your repository", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "namespace", + Usage: "Required, Kubernetes namespace to deploy Kontinuous", + Value: "kontinuous", + }, + cli.StringFlag{ + Name: "repository, repo, r", + Usage: "Required, Github repository name ", + }, + cli.StringFlag{ + Name: "owner, o", + Usage: "Required, Github owner name", + }, + }, + + Action: initialize, + }, } app.Run(os.Args) } @@ -364,23 +377,19 @@ func deleteBuild(c *cli.Context) { func deployApp(c *cli.Context) { namespace := c.String("namespace") - accessKey := c.String("s3-access-key") - secretKey := c.String("s3-secret-key") authCode := c.String("auth-secret") clientId := c.String("github-client-id") clientSecret := c.String("github-client-secret") missingFields := false - if namespace == "" || accessKey == "" || secretKey == "" || authCode == "" || clientId == "" || clientSecret == "" { - fmt.Println("missing required fields!") + if namespace == "" || authCode == "" || clientId == "" || clientSecret == "" { missingFields = true - } if !missingFields { - err := DeployKontinuous(namespace, accessKey, secretKey, authCode, clientId, clientSecret) + err := DeployKontinuous(namespace, authCode, clientId, clientSecret) if err != nil { - fmt.Println("Oops something went wrong. Unable to deploy kontinuous.") + fmt.Println("Missing fields. Unable to deploy Kontinuous.") fmt.Println(err) os.Exit(1) } @@ -388,14 +397,40 @@ func deployApp(c *cli.Context) { } } +func initialize(c *cli.Context) { + config, err := apiReq.GetConfigFromFile(c.GlobalString("conf")) + if err != nil { + os.Exit(1) + } + namespace := c.String("namespace") + owner := c.String("owner") + repository := c.String("repository") + token := config.Token + + missingFields := false + if namespace == "" || owner == "" || repository == "" { + missingFields = true + } + + if !missingFields { + err := Init(namespace, owner, repository, token) + if err != nil { + fmt.Println("Missing fields. Unable to initialize Kontinuous in the repository") + fmt.Println(err) + os.Exit(1) + } + fmt.Println("Success! Initialization complete.") + } +} + func removeDeployedApp(c *cli.Context) { err := RemoveResources() if err != nil { - fmt.Println("Oops something went wrong. Unable to remove kontinuous.") + fmt.Println("Unable to remove kontinuous.") fmt.Println(err) os.Exit(1) } - fmt.Println("Success! Kontinuous app has been removed from the cluster. ") + fmt.Println("Success! Kontinuous resources has been removed from the cluster. ") } diff --git a/cli/deploy.go b/cli/deploy.go index c3192ca..1b7e3b1 100644 --- a/cli/deploy.go +++ b/cli/deploy.go @@ -3,6 +3,7 @@ package main import ( "bytes" "encoding/base64" + "errors" "fmt" "os" "os/exec" @@ -11,6 +12,14 @@ import ( "time" ) +var namespaceData = ` +--- +kind: Namespace +apiVersion: v1 +metadata: + name: {{.Namespace}} +` + var secretData = ` { "AuthSecret": "{{.AuthCode}}", @@ -22,7 +31,20 @@ var secretData = ` ` -var minio = ` +var secret = ` + +--- +kind: Secret +apiVersion: v1 +metadata: + name: kontinuous-secrets + namespace: {{.Namespace}} +data: + kontinuous-secrets: {{.SecretData}} + +` + +var minioSvc = ` --- kind: Service apiVersion: v1 @@ -40,6 +62,8 @@ spec: - name: service port: 9000 targetPort: 9000 +` +var minioRc = ` --- kind: ReplicationController apiVersion: v1 @@ -68,11 +92,6 @@ spec: - name: minio image: minio/minio:latest imagePullPolicy: Always - env: - - name: MINIO_ACCESS_KEY - value: {{.AccessKey}} - - name: MINIO_SECRET_KEY - value: {{.SecretKey}} args: - /data volumeMounts: @@ -87,20 +106,7 @@ spec: timeoutSeconds: 1 ` -var secret = ` - ---- -kind: Secret -apiVersion: v1 -metadata: - name: kontinuous-secrets - namespace: {{.Namespace}} -data: - kontinuous-secrets: {{.SecretData}} - -` - -var etcd = ` +var etcdSvc = ` --- kind: Service apiVersion: v1 @@ -118,6 +124,9 @@ spec: - name: api port: 2379 targetPort: 2379 +` + +var etcdRc = ` --- kind: ReplicationController apiVersion: v1 @@ -152,7 +161,7 @@ spec: containerPort: 2379 ` -var registry = ` +var registrySvc = ` --- kind: Service apiVersion: v1 @@ -170,6 +179,9 @@ spec: - name: service port: 5000 targetPort: 5000 +` + +var registryRc = ` --- kind: ReplicationController apiVersion: v1 @@ -201,7 +213,7 @@ spec: ` -var kontinuousService = ` +var kontinuousSvc = ` --- kind: Service apiVersion: v1 @@ -222,7 +234,7 @@ spec: targetPort: 3005 ` -var kontinuousRC = ` +var kontinuousRc = ` --- kind: ReplicationController apiVersion: v1 @@ -282,7 +294,6 @@ metadata: spec: ports: - name: dashboard - nodePort: 30345 port: 5000 protocol: TCP targetPort: 5000 @@ -345,11 +356,6 @@ type Deploy struct { GHSecret string } -const ( - KONTINUOUS_SPECS_FILE = "/tmp/kontinuous-specs.yml" - KONTINUOUS_RC_SPEC_FILE = "/tmp/kontinuous-rc-spec.yml" -) - func generateResource(templateStr string, deploy *Deploy) (string, error) { template := template.New("kontinuous Template") @@ -375,7 +381,7 @@ func saveToFile(path string, data ...string) error { defer file.Close() } - file, err = os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0644) + file, err = os.OpenFile(path, os.O_RDWR, 0644) if err != nil { fmt.Println(err.Error()) return err @@ -413,12 +419,21 @@ func createKontinuousResouces(path string) error { return nil } -func deleteKontinuousResources(path string) error { - cmd := fmt.Sprintf("kubectl delete -f %s", path) +func deleteKontinuousResources() error { + //remove namespace file + fmt.Println("Removing Kontinuous resources ... ") + cmd := fmt.Sprintf("rm -f /tmp/deploy/APP_NAMESPACE.yml") _, err := exec.Command("bash", "-c", cmd).Output() + + if err != nil { + fmt.Println("Unable to remove namespace resource") + } + cmd = fmt.Sprintf("kubectl delete -f %s", "/tmp/deploy/") + result, err := exec.Command("bash", "-c", cmd).Output() if err != nil { return err } + fmt.Printf("%s", string(result)) return nil } @@ -443,59 +458,127 @@ func fetchKontinuousIP(serviceName, namespace string) (string, error) { return ip, nil } -func RemoveResources() error { - err := deleteKontinuousResources(KONTINUOUS_SPECS_FILE) +func deployResource(definition string, fileName string, deploy *Deploy) error { + resource, _ := generateResource(definition, deploy) + resourceArr := strings.Split(fileName, "_") + resourceName := resourceArr[0] + resourceType := resourceArr[1] + + cmd := fmt.Sprintf("mkdir -p %s", "/tmp/deploy") + _, _ = exec.Command("bash", "-c", cmd).Output() + + filePath := fmt.Sprintf("/tmp/deploy/%s.yml", fileName) + saveToFile(filePath, resource) + //check if it exist + cmd = fmt.Sprintf("kubectl get -f %s", filePath) + _, err := exec.Command("bash", "-c", cmd).Output() + if err == nil { + return errors.New(fmt.Sprintf("%s: %s already exists \n", resourceType, resourceName)) + } + + err = createKontinuousResouces(filePath) if err != nil { + fmt.Sprintf("Unable to deploy resource: %s \n", err) return err } - err = deleteKontinuousResources(KONTINUOUS_RC_SPEC_FILE) + fmt.Printf("Successfully deployed %s - %s \n", resourceName, resourceType) + return nil +} + +func getS3Details(deploy *Deploy) error { + + var podName string + cmd := fmt.Sprintf(`kubectl get po --namespace=%s -l app=minio,type=object-store --no-headers | awk '{print $1}'`, deploy.Namespace) + waitingTime := 0 + + for len(podName) == 0 && waitingTime < 30 { + pod, _ := exec.Command("bash", "-c", cmd).Output() + podName = string(pod) + time.Sleep(2 * time.Second) + waitingTime += 2 + } + + if len(podName) == 0 { + return errors.New("Unable to Deploy Kontinuous. Dependency Minio Storage is unavailable") + } + + podName = strings.TrimSpace(podName) + cmd = fmt.Sprintf(`kubectl logs --namespace=%s %s | grep AccessKey | awk '{print $2}'`, deploy.Namespace, podName) + s3AccessKey, err := exec.Command("bash", "-c", cmd).Output() + if err != nil { + fmt.Println(err.Error()) + return errors.New("Unable to Deploy Kontinuous. Dependency Minio Storage Access Key is unavailable") + } + + deploy.AccessKey = strings.TrimSpace(string(s3AccessKey)) + cmd = fmt.Sprintf(`kubectl logs --namespace=%s %s | grep AccessKey | awk '{print $4}'`, deploy.Namespace, podName) + s3AccessSecret, err := exec.Command("bash", "-c", cmd).Output() + if err != nil { + return errors.New("Unable to Deploy Kontinuous. Dependency Minio Storage Secret Key is unavailable") + } + + deploy.SecretKey = strings.TrimSpace(string(s3AccessSecret)) + return nil +} + +func RemoveResources() error { + err := deleteKontinuousResources() if err != nil { + fmt.Printf("Unable to remove Kontinuous resources. %s \n", err.Error()) return err } return nil } -func DeployKontinuous(namespace, accesskey, secretkey, authcode, clientid, clientsecret string) error { - fmt.Println("Deploying Kontinuous...") +func DeployKontinuous(namespace, authcode, clientid, clientsecret string) error { + fmt.Println("Deploying Kontinuous resources ...") deploy := Deploy{ Namespace: namespace, - AccessKey: accesskey, - SecretKey: secretkey, AuthCode: authcode, GHClient: clientid, GHSecret: clientsecret, } - sData, _ := generateResource(secretData, &deploy) - deploy.SecretData = encryptSecret(sData) - secret, _ := generateResource(secret, &deploy) - minioStr, _ := generateResource(minio, &deploy) - etcdStr, _ := generateResource(etcd, &deploy) - registryStr, _ := generateResource(registry, &deploy) - kontinuousSvcStr, _ := generateResource(kontinuousService, &deploy) - dashboardSvcStr, _ := generateResource(dashboardSvc, &deploy) - - //save string in a file - saveToFile(KONTINUOUS_SPECS_FILE, secret, minioStr, etcdStr, registryStr, kontinuousSvcStr, dashboardSvcStr) - err := createKontinuousResouces(KONTINUOUS_SPECS_FILE) + deployResource(namespaceData, "APP_NAMESPACE", &deploy) + deployResource(minioSvc, "MINIO_SVC", &deploy) + deployResource(minioRc, "MINIO_RC", &deploy) + deployResource(etcdSvc, "ETCD_SVC", &deploy) + deployResource(etcdRc, "ETCD_RC", &deploy) + deployResource(registrySvc, "REGISTRY_SVC", &deploy) + deployResource(registryRc, "REGISTRY_RC", &deploy) + deployResource(kontinuousSvc, "KONTINUOUS_SVC", &deploy) + deployResource(dashboardSvc, "DASHBOARD_SVC", &deploy) + + err := getS3Details(&deploy) if err != nil { - return err + fmt.Println(err.Error()) } + sData, _ := generateResource(secretData, &deploy) + deploy.SecretData = encryptSecret(sData) + deployResource(secret, "APP_SECRET", &deploy) + ip, _ := fetchKontinuousIP("kontinuous", deploy.Namespace) dashboardIp, _ := fetchKontinuousIP("kontinuous-ui", deploy.Namespace) deploy.DashboardIP = dashboardIp deploy.KontinuousIP = ip - kontinuousRcStr, _ := generateResource(kontinuousRC, &deploy) - dashboardRcStr, _ := generateResource(dashboardRc, &deploy) - - saveToFile(KONTINUOUS_RC_SPEC_FILE, kontinuousRcStr, dashboardRcStr) - err = createKontinuousResouces(KONTINUOUS_RC_SPEC_FILE) + err = deployResource(kontinuousRc, "KONTINUOUS_RC", &deploy) if err != nil { + fmt.Println("Unable to deploy Kontinuous Deploy Replication Controller \n %s", err.Error()) return err } + err = deployResource(dashboardRc, "DASHBOARD_RC", &deploy) + if err != nil { + fmt.Printf("Unable to deploy Kontinuous UI Replication Controller \n %s", err.Error()) + return err + } + + fmt.Println("_____________________________________________________________\n") + fmt.Printf("Kontinuous API : http://%s:8080 \n", deploy.KontinuousIP) + fmt.Printf("Kontinuous Dashboard : http://%s:5000 \n", deploy.DashboardIP) + fmt.Println("_____________________________________________________________\n") return nil } diff --git a/cli/init.go b/cli/init.go new file mode 100644 index 0000000..e8d382f --- /dev/null +++ b/cli/init.go @@ -0,0 +1,106 @@ +package main + +import ( + "bytes" + b64 "encoding/base64" + "errors" + "fmt" + "github.com/AcalephStorage/kontinuous/api" + "net/http" + "text/template" +) + +var initDefinition = ` +--- +apiVersion: v1alpha1 +kind: Pipeline +metadata: + name: {{.ProjectName}} + namespace: {{.Namespace}} +spec: + selector: + matchLabels: + app: {{.Namespace}} + template: + metadata: + name: {{.Namespace}} + labels: + app: {{.ProjectName}} + stages: + - name: Build Image + type: docker_build +` + +type InitDefinition struct { + ProjectName string + Namespace string +} + +func generateInitResource(templateStr string, init *InitDefinition) (string, error) { + + template := template.New("kontinuous init Template") + template, _ = template.Parse(templateStr) + var b bytes.Buffer + + err := template.Execute(&b, init) + + if err != nil { + fmt.Println(err.Error()) + } + + return b.String(), nil + +} + +func generateInit(token, owner, repo, namespace string) error { + + initDef := InitDefinition{ + Namespace: namespace, + ProjectName: repo, + } + + def, err := generateInitResource(initDefinition, &initDef) + if err != nil { + fmt.Printf("error", err.Error()) + return err + } + + repoPath := fmt.Sprintf("/repos/%s/%s/contents/.pipeline.yml", owner, repo) + + //check if pipeline already exists + client := http.DefaultClient + contents, _ := api.SendGithubRequest(token, client, "GET", repoPath, nil) + + if contents != nil { + return errors.New(".pipeline.yml already exists") + } + + sEnc := b64.StdEncoding.EncodeToString([]byte(def)) + + contentData := fmt.Sprintf(`{ + "committer":{ + "name":"kontinuous", + "email":"admin@kontinuous.sg" + }, + "message": "Commit Initial Pipeline", + "content": "%s"}`, sEnc) + + data := []byte(contentData) + _, err = api.SendGithubRequest(token, client, "PUT", repoPath, data) + if err != nil { + fmt.Printf("Error: %s", err.Error()) + return err + } + fmt.Println(".pipeline.yml is now available in the repository.") + return nil +} + +func Init(namespace, owner, repo, token string) error { + + fmt.Printf("Initializing Kontinuous in repository: %s/%s \n", owner, repo) + err := generateInit(token, owner, repo, namespace) + if err != nil { + return err + } + return nil +} From 85ffd27b1366c6e83de9d842f553bbb2920fd868 Mon Sep 17 00:00:00 2001 From: Stefany Serino Date: Thu, 19 May 2016 15:46:53 +0800 Subject: [PATCH 17/30] Add endpoint to create definition file updateDefinition endpoint will create the file when sha/blob is not provided --- api/pipeline.go | 9 +++--- pipeline/common_test.go | 4 +++ pipeline/definition.go | 62 +++++++++++++++++++++++++++++++++++++++++ pipeline/pipeline.go | 49 ++------------------------------ scm/client.go | 1 + scm/github/github.go | 19 +++++++++++++ 6 files changed, 92 insertions(+), 52 deletions(-) diff --git a/api/pipeline.go b/api/pipeline.go index ce8f21a..95bb5b3 100644 --- a/api/pipeline.go +++ b/api/pipeline.go @@ -82,8 +82,8 @@ func (p *PipelineResource) Register(container *restful.Container) { Writes(ps.DefinitionFile{}). Filter(requireAccessToken)) - ws.Route(ws.PUT("/{owner}/{repo}/definition").To(p.updateDefinition). - Doc("Update definition file of the pipeline"). + ws.Route(ws.POST("/{owner}/{repo}/definition").To(p.updateDefinition). + Doc("Update definition file of the pipeline, creates one if it does not exist"). Operation("updateDefinition"). Param(ws.PathParameter("owner", "repository owner name").DataType("string")). Param(ws.PathParameter("repo", "repository name").DataType("string")). @@ -207,11 +207,10 @@ func (p *PipelineResource) updateDefinition(req *restful.Request, res *restful.R client := newSCMClient(req) body, _ := ioutil.ReadAll(req.Request.Body) - type definitionPayload struct { + payload := new(struct { Definition *ps.DefinitionFile `json:"definition"` Commit map[string]string `json:"commit"` - } - payload := &definitionPayload{} + }) if err := json.Unmarshal(body, &payload); err != nil { jsonError(res, http.StatusInternalServerError, err, "Unable to read request payload") return diff --git a/pipeline/common_test.go b/pipeline/common_test.go index 32a9e09..dd23dd5 100644 --- a/pipeline/common_test.go +++ b/pipeline/common_test.go @@ -235,6 +235,10 @@ func (s MockSCMClient) GetContents(owner, repo, path, ref string) (*scm.Reposito }, true } +func (s MockSCMClient) CreateFile(owner, repo, path, message, branch string, content []byte) (*scm.RepositoryContent, error) { + return &scm.RepositoryContent{}, nil +} + func (s MockSCMClient) UpdateFile(owner, repo, path, blob, message, branch string, content []byte) (*scm.RepositoryContent, error) { return &scm.RepositoryContent{}, nil } diff --git a/pipeline/definition.go b/pipeline/definition.go index c87f57e..f0d54ad 100644 --- a/pipeline/definition.go +++ b/pipeline/definition.go @@ -2,10 +2,15 @@ package pipeline import ( "errors" + "fmt" + "time" + "encoding/base64" "encoding/json" "github.com/ghodss/yaml" + + "github.com/AcalephStorage/kontinuous/scm" ) type ( @@ -47,6 +52,63 @@ func (d *Definition) GetStages() []*Stage { return stages } +func (d *DefinitionFile) SaveToRepo(c scm.Client, owner, repo string, commit map[string]string) (*DefinitionFile, error) { + source, exists := c.GetRepository(owner, repo) + if !exists { + return nil, fmt.Errorf("Unable to find repository %s/%s", owner, repo) + } + defaultBranch := source.DefaultBranch + branch := defaultBranch + + if commit["option"] == "pull_request" { + branch = commit["branch_name"] + if len(branch) == 0 { + return nil, fmt.Errorf("Branch name not provided") + } + + head, err := c.GetHead(owner, repo, defaultBranch) + if err != nil { + return nil, err + } + _, err = c.CreateBranch(owner, repo, branch, head) + if err != nil { + return nil, err + } + // wait for branch to be created (todo: verify existence of branch) + time.Sleep(3 * time.Second) + } + + decodedContent, err := base64.URLEncoding.DecodeString(*d.Content) + if err != nil { + return nil, err + } + + file := &scm.RepositoryContent{} + if d.SHA != nil { + if len(commit["message"]) == 0 { + commit["message"] = fmt.Sprintf("Update %s", PipelineYAML) + } + file, err = c.UpdateFile(owner, repo, PipelineYAML, *d.SHA, commit["message"], branch, decodedContent) + } else { + if len(commit["message"]) == 0 { + commit["message"] = fmt.Sprintf("Create %s", PipelineYAML) + } + file, err = c.CreateFile(owner, repo, PipelineYAML, commit["message"], branch, decodedContent) + } + if err != nil { + return nil, err + } + + if commit["option"] == "pull_request" { + err = c.CreatePullRequest(owner, repo, defaultBranch, branch, commit["message"]) + if err != nil { + return nil, err + } + } + + return &DefinitionFile{SHA: file.SHA}, nil +} + func GetDefinition(definition []byte) (payload *Definition, err error) { if len(definition) == 0 { diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index 4eeb0ca..5ccb2a1 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -536,54 +536,9 @@ func (p *Pipeline) GetDefinitionFile(c scm.Client, ref string) (*DefinitionFile, } // UpdateDefinitionFile commits the changes of the definition file (PipelineYAML) +// or creates the file if it does not exist // either directly to the default branch // or through a pull request func (p *Pipeline) UpdateDefinitionFile(c scm.Client, file *DefinitionFile, commit map[string]string) (*DefinitionFile, error) { - source, exists := c.GetRepository(p.Owner, p.Repo) - if !exists { - return nil, fmt.Errorf("Unable to find repository %s/%s", p.Owner, p.Repo) - } - defaultBranch := source.DefaultBranch - branch := defaultBranch - - message := commit["message"] - if len(message) == 0 { - message = fmt.Sprintf("Update %s", PipelineYAML) - } - - if commit["option"] == "pull_request" { - branch = commit["branch_name"] - if len(branch) == 0 { - return nil, fmt.Errorf("Branch name not provided") - } - - head, err := c.GetHead(p.Owner, p.Repo, defaultBranch) - if err != nil { - return nil, err - } - _, err = c.CreateBranch(p.Owner, p.Repo, branch, head) - if err != nil { - return nil, err - } - // wait for branch to be created (todo: verify existence of branch) - time.Sleep(3 * time.Second) - } - - decodedContent, err := base64.URLEncoding.DecodeString(*file.Content) - if err != nil { - return nil, err - } - updatedFile, err := c.UpdateFile(p.Owner, p.Repo, PipelineYAML, *file.SHA, message, branch, decodedContent) - if err != nil { - return nil, err - } - - if commit["option"] == "pull_request" { - err = c.CreatePullRequest(p.Owner, p.Repo, defaultBranch, branch, message) - if err != nil { - return nil, err - } - } - - return &DefinitionFile{SHA: updatedFile.SHA}, nil + return file.SaveToRepo(c, p.Owner, p.Repo, commit) } diff --git a/scm/client.go b/scm/client.go index 833d739..0818649 100644 --- a/scm/client.go +++ b/scm/client.go @@ -46,6 +46,7 @@ type Client interface { CreateStatus(owner, repo, sha string, stageID int, stageName, state string) error GetFileContent(owner, repo, path, ref string) ([]byte, bool) GetContents(owner, repo, path, ref string) (*RepositoryContent, bool) + CreateFile(owner, repo, path, message, branch string, content []byte) (*RepositoryContent, error) UpdateFile(owner, repo, path, blob, message, branch string, content []byte) (*RepositoryContent, error) GetRepository(owner, repo string) (*Repository, bool) ListRepositories(user string) ([]*Repository, error) diff --git a/scm/github/github.go b/scm/github/github.go index eb256e8..fa2c23d 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -109,6 +109,25 @@ func (gc *Client) GetContents(owner, repo, path, ref string) (*scm.RepositoryCon }, true } +// CreateFile commits a new file to a repository +func (gc *Client) CreateFile(owner, repo, path, message, branch string, content []byte) (*scm.RepositoryContent, error) { + if len(message) == 0 { + message = fmt.Sprintf("Create %s", path) + } + resp, _, err := gc.client().Repositories.CreateFile(owner, + repo, + path, + &github.RepositoryContentFileOptions{ + Message: &message, + Content: content, + Branch: &branch, + }) + if err != nil { + return nil, err + } + return &scm.RepositoryContent{SHA: resp.Content.SHA}, nil +} + // UpdateFile commits diff of a file content from the given blob func (gc *Client) UpdateFile(owner, repo, path, blob, message, branch string, content []byte) (*scm.RepositoryContent, error) { if len(message) == 0 { From 2477eda6cdfbfce8b69c287b0cdd5abf1bdb7871 Mon Sep 17 00:00:00 2001 From: dexter Date: Thu, 19 May 2016 16:18:29 +0800 Subject: [PATCH 18/30] updated docs --- README.md | 260 +----------------- docs/api.md | 65 +++++ docs/pipeline.md | 177 ++++++++++++ docs/setup.md | 78 ++++++ ...c.yml.example.yaml => k8s-spec.yml.example | 0 5 files changed, 332 insertions(+), 248 deletions(-) create mode 100644 docs/api.md create mode 100644 docs/pipeline.md create mode 100644 docs/setup.md rename k8s-spec.yml.example.yaml => k8s-spec.yml.example (100%) diff --git a/README.md b/README.md index cb6cdac..3f36180 100644 --- a/README.md +++ b/README.md @@ -46,272 +46,36 @@ Parameters: | --github-client-id | The Github client ID provided when registering kontinuous as a Github OAuth application | | --github-client-secret | The Github client secret provided when registering kontinuous as a Github OAuth application | -This will launch the following via `kubectl`: - - etcd - - minio - - docker registry - - kontinuous - - kontinuous-ui +This will launch `kontinuous` via the locally configured `kubectl` in the given namespace together with `etcd`, `minio`, a docker `registry`, and `kontinuous-ui`. This expects that the kubernetes cluster supports the LoadBalancer service. -This will launch `kontinuous` via the locally configured `kubectl` in the given namespace together with `etcd`, `minio`, a docker `registry`, and the Kontinuous UI. This expects that the kubernetes cluster supports the LoadBalancer service. +Once a public IP for `kontinuous-ui` is available, the Github OAuth Application settings needs to be modified to reflect the actual IP address of `kontinuous-ui` for the Homepage and Callback URL. -Alternatively, for more customization, a sample yaml file for running kontinuous and its dependencies in Kubernetes can be found [here](./k8s-spec.yml.example). See [below](#running-in-kubernetes) for how to configure secrets. +Alternatively, for more customization, a sample yaml file for running kontinuous and its dependencies in Kubernetes can be found [here](./k8s-spec.yml.example). More details can be found [here](docs/setup.md). -Once running, add a [.pipeline.yml](#pipeline-spec) to the root of your Github repo and configure the webhooks. +Once running, add a [.pipeline.yml](#pipeline-spec) to the root of your Github repo and configure the webhooks. More details about pipeline spec creation can be found [here](docs/pipeline.md). Example pipelines can be found in [/examples](./examples) The [CLI client](#clients) or [API](#api) can be used to view build status or logs. -### Dependencies - -Running kontinuous requires the following to be setup: - - - **etcd** - - `etcd` is used as a backend for storing pipeline and build details. This is a dedicated instance to avoid poluting with the Kubernetes etcd cluster. - - - **minio** - - `minio` is used to store the logs and artifacts. S3 could also be used as it is compatible with `minio`, although this has not been tested yet. - -- **docker registry** - - `registry` is used to store internal docker images. - -- **kubernetes** - - Kontinuous uses Kubernetes Jobs heavily so will require at least version 1.1 with Jobs enabled - - -### Running in Kubernetes - -Kontinuous is meant to run inside a kubernetes cluster, preferrably by a Deployment or Replication Controller. - -The docker image can be found here: [quay.io/acaleph/kontinuous](https://quay.io/acaleph/kontinuous) - -The following environment variables needs to be defined: - -| Environment Variable | Description | Example | -|----------------------|-----------------------------------------|------------------------| -| KV_ADDRESS | The etcd address | etcd:2379 | -| S3_URL | The minio address | http://minio:9000 | -| KONTINUOUS_URL | The address where kontinuous is running | http://kontinuous:3005 | -| INTERNAL_REGISTRY | The internal registry address | internal-registry:5000 | - -A Kubernetes Secret also needs to be defined and mounted to the Pod. The secret should have a key named `kontinuous-secrets` and should contain the following data (must be base64 encoded): - -```json -{ - "AuthSecret": "base64 encoded auth secret", - "S3SecretKey": "s3 secret key", - "S3AccessKey": "s3 access key", - "GithubClientID": "github client ID", - "GithubClientSecret": "github client secret" -} -``` - -`AuthSecret` is the secret for authenticating requests. This is needed by the clients to communicate with kontinuous through JWT. - -`S3SecretKey` and `S3AccessKey` are the keys needed to access minio (or S3). - -`GithubClientID` and `GithubClientSecret` are optional and only required for Github only authentication. (See below) - -The secret needs to be mounted to the Pod to the path `/.secret`. - -## Using Kontinuous - -### Preparing the repository - -#### Pipeline Spec - -The repository needs to define a build pipeline in the repository root called `.pipeline.yml` - -Here's a sample `.pipeline.yml`: - -```yaml ---- -apiVersion: v1alpha1 -kind: Pipeline -metadata: - name: kontinuous - namespace: acaleph -spec: - selector: - matchLabels: - app: kontinuous - type: ci-cd - template: - metadata: - name: kontinuous - labels: - app: kontinuous - type: ci-cd - notif: - - type: slack - metadata: - url: slackurl #taken from secret - username: slackuser #taken from secret - channel: slackchannel #taken from secret - secrets: - - notifcreds - - docker-credentials - stages: - - name: Build Docker Image - type: docker_build - - name: Unit Test - type: command - params: - command: - - make - - test - - name: Publish to Quay - type: docker_publish - params: - external_registry: quay.io - external_image_name: acaleph/kontinuous - require_credentials: "TRUE" - username: user # taken from secret - password: password # taken from secret - email: email # taken from secret -``` - -The format is something similar to K8s Specs. Here are more details on some of the fields: - - - `namespace` - the namespace to run the build - - `matchLabels`/`labels` - the labels that are used for building the job - - `stages` - defines the stages of the build pipeline - -The general definition of a stage is: - -```yaml -name: Friendly name -type: {docker_build,command,docker_publish} -params: - key: value -``` - -- `type` can be: `docker_build`, `docker_publish`, `command`, or `deploy`. -- `params` is a map of parameters to be loaded as environment variables. - -#### Notification - -- `type` can be: `slack`. -- `metadata` is a map of values needed for a certain notification type. The metadata value should be a **key** from one of the secrets file defined - - `metadata` of notification `type=slack` has the following keys: - - - `url` is a slack incoming messages webhook url. - - `channel` is optional. If set, it will override default channel - - `username` is optional. If set, it will override default username - -In the future releases, kontinuous notification will support other notification services. e.g. email, hipchat, etc. - -### Secrets - -- `secrets` is a list of secrets that will be used as values for stages and notification. - -#### Stages - -`docker_build` builds a Docker Image and pushes the images to a internal registry. It can work without additional params. By default, it uses the `Dockerfile` inside the repository root. Optional params are: - - - `dockerfile_path` - the path where the Dockerfile is located - - `dockerfile_name` - the file name of the Dockerfile - -After a build, the image is stored inside the internal docker registry. - -`docker_publish` pushes the previously build Docker image to an external registry. It requires the following params: - - - `external_registry` - the external registry name (eg. quay.io) - - `external_image_name` - the name of the image (eg. acaleph/kontinuous) - -Optional params: - - - `require_crendentials` - defaults to `false`. Set to `true` if registry requires authentication - -Required secrets: - - - `dockeruser` - - `dockerpass` - - `dockeremail` - - These secrets needs to be defined in at least one of the secrets provided. - -The image that will be pushed is the image that was previously built. This does not work for now if no image was created. - -`command` runs a command on the newly create docker image or on the image specified. Required param is `command` which is a list of string defining the command to execute. - -Optional params are: - - - `args` - a list of string to serve as the arguments for the command - - `image` - the image to run the commands in. If not specified, the previous built image will be used. - - `dependencies` - a list of Kubernetes spec files to run as dependencies for running the command. Useful when running integration tests. - -`deploy` deploys a Kubernetes Spec file (yaml) to kubernetes. - ---- -# TODO: WIP. This section is about accessing the API. not necessary if using the UI. Might move this to a separate section...(?) ---- - -### Authentication - -There are currently two ways of authenticating with kontinuous. One is by using Github's OAuth Web Application Flow and the other one is with JSON Web Tokens. - -#### Github OAuth - -This is the authorization process used by `kontinuous-ui`. This is a 3 step process detailed [here](https://developer.github.com/v3/oauth/#web-application-flow) with a slightly different variation: - -1. Kontinuous needs to be registered as a Github OAuth Application [here](https://github.com/settings/applications/new). - -2. Redirect users to request Github Access (step 1 in web application flow): - -``` -GET https://github.com/login/oauth/authorize -``` - -3. Send authorization code to Kontinuous: - -``` -POST {kontinuous-url}/api/v1/login/github?code={auth_code}&state={state_from_step_1} -``` - -This should return a JSON Web Token in the body that can be used to authenticate further requests. - - -#### JSON Web Token - -Currently, only Github Repositories are supported. A github token needs to be generated in order to access the repositories. - -To generate a github token, follow this [link](https://github.com/settings/tokens/new). +## Clients -Make sure to enable access to the following: +There are two clients currently available: - - repo - - admin:repo_hook - - user +### Kontinuous CLI -The script `scripts/jwt-gen` can generate a JSON Web Token to be used for authentication with Kontinuous. +The CLI tool is the one that is used in the gettings started section. It can bootstrap Kontinuous to a running Kubernetes Cluster and can access details on Kontinuous pipelines and builds. -```console -$ scripts/jwt-gen --secret {base64url encoded secret} --github-token {github-token} -``` +More info about the CLI can be found [here](https://github.com/AcalephStorage/kontinuous/tree/develop/cli) and the binary can be downloaded [here](https://github.com/AcalephStorage/kontinuous/releases). -This generates a JSON Web Token and can be added to the request header as `Authorization: Bearer {token}` to authenticate requests. +### Kontinuous UI -The generated token's validity can be verified at [jwt.io](https://jwt.io). +Kontinuous UI is a web based client for Kontinuous. Bootstrapping Kontinuous using the CLI will install the UI on the Kubernetes Cluster too. More info about the UI can be found [here](https://github.com/AcalephStorage/kontinuous-ui). ## API -kontinuous is accessible from it's API and docs can be viewed via Swagger. - -The API doc can be accessed via `{kontinuous-address}/apidocs` - -## Clients - -At the moment, there is a basic cli client binary [here](https://github.com/AcalephStorage/kontinuous/releases) and code available [here](https://github.com/AcalephStorage/kontinuous/tree/develop/cli). - -A Web based Dashboard is under development. +Kontinuous is accessible from it's API and docs can be viewed via Swagger. More details about using the API and Authentication can be found [here](docs/api.md). ## Development diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..46cc9d4 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,65 @@ +Kontinuous API +============== + +Kontinuous provides an API to access pipeline and build details. + +## Swagger + +The API doc can be accessed from a running kontinuous instance via `{kontinuous-address}/apidocs`. + +## Authentication + +Kontinuous API uses JSON Web Tokens for authentication. This is useful when accessing the API directly. There are several ways of getting a token. + +The token needs to be added to the header as `Authorization: Bearer {token}` to authenticate requests. + +### Github + +This is the method used by Kontinuous UI. This is a slightly modified [Github's OAuth Web Application Flow](https://developer.github.com/v3/oauth/#web-application-flow). + +1. Kontinuous needs to be registered as a [Github OAuth Application](https://github.com/settings/applications/new). The Client ID and Secret needs to be defined in the kontinuous secret. + +2. Redirect users to request Github Access (step 1 in web application flow): + +``` +GET https://github.com/login/oauth/authorize?client_id={clientid}&redirect_uri={redirect_url}& scope=user,repo,admin:repo_hook&state={random string} +``` + +3. Send authorization code to Kontinuous: + +``` +POST {kontinuous-url}/api/v1/login/github?code={auth_code}&state={state_from_step_1} +``` + +This will return a JSON Web Token that can be used to access the API. + +### Auth0 + +[Auth0](https://auth0.com) is a service for managing authentications. This can be used to generate an auth secret and provide Github access for kontinuous. + +1. Use Auth0 to create an auth secret to be added to Kontinuous. +2. The Web Interface needs to authenticate against Auth0 to use Kontinuous. + +Auth0 will provide the JSON Web Token. + +### JWT Creation + +A JSON Web Token can be manually created. This requires a github token and the auth secret used by Kontinuous. + +To generate a github token, follow this [link](https://github.com/settings/tokens/new). + +Make sure to enable access to the following: + + - repo + - admin:repo_hook + - user + +The script `scripts/jwt-gen` can generate a JSON Web Token to be used for authentication with Kontinuous. + +```console +$ scripts/jwt-gen --secret {secret} --github-token {github-token} +``` +The generated token's validity can be verified at [jwt.io](https://jwt.io). + + + diff --git a/docs/pipeline.md b/docs/pipeline.md new file mode 100644 index 0000000..1307660 --- /dev/null +++ b/docs/pipeline.md @@ -0,0 +1,177 @@ +Kontinuous Pipelines +==================== + +A repository needs to define a pipeline spec by adding `.pipeline.yml` to the root directory of the repository. + +## Pipeline Specification + +Here's an example pipeline specification. + +```yaml +--- +apiVersion: v1alpha1 +kind: Pipeline +metadata: + name: kontinuous + namespace: acaleph +spec: + selector: + matchLabels: + app: kontinuous + type: ci-cd + template: + metadata: + name: kontinuous + labels: + app: kontinuous + type: ci-cd + notif: + - type: slack + secrets: + - notifcreds + - docker-credentials + stages: + - name: Build Docker Image + type: docker_build + - name: Unit Test + type: command + params: + command: + - make + - test + - name: Publish to Quay + type: docker_publish + params: + external_registry: quay.io + external_image_name: acaleph/kontinuous + require_credentials: "TRUE" + username: user # taken from secret + password: password # taken from secret + email: email # taken from secret +``` + +Some important fields on the spec: + +| Field | Description | +|-----------------------|-----------------------------------------------------------| +| metadata.namespace | Defines which namespace to run the builds of the pipeline | +| spec.template.notif | Defines the notification used by this pipeline | +| spec.template.secrets | Defines the secrets used by this pipeline | +| spec.stages | Defines the build stages | + +### Notification + +Currently only slack is supported. More notifiers will be added in the future. + +```yaml +spec: + template: + notif: + - type: slack +``` + +Slack details are taken from the secrets. The following secrets needs to be defined: + +| secret name | details | +|--------------|------------------------------------------------| +| slackchannel | the channel to post the notifications | +| slackurl | the slack url | +| slackuser | the user to display when showing notifications | + +### Secrets + +Kubernetes secrets can be added to the pipeline. Each of the secret entries will be added as environment variables. + +```yaml +spec: + template: + secrets: + - secret1 + - secret2 +``` + +### Stages + +Stages are the build definitions. Currently there are four different stage types. + +```yaml +spec: + template: + stages: + - name: Friendly name + type: docker_build + params: {} +``` + +| Stage | Description | +|----------------|-------------------------------------------------------------------| +| docker_build | build a docker image | +| docker_publish | publish a docker image to an external registry | +| command | run commands against a previously built image or a specific image | +| deploy | deploys a kubernetes spec file to kubernetes | + +#### docker_build + +Builds a Docker Image and pushes the images to the internal registry. It can work without additional params. By default, it uses the `Dockerfile` inside the repository root. + +Optional params are: + +| Parameter | Description | +|-----------------|------------------------------------------| +| dockerfile_path | the path where the Dockerfile is located | +| dockerfile_name | the file name of the Dockerfile | + +#### docker_publish + +pushes the previously build Docker image to an external registry + +Required Params: + +| Parameter | Description | +|---------------------|--------------------------------------------------| +| external_registry | the external registry name (eg. quay.io) | +| external\_image\_name | the name of the image (eg. acaleph/kontinuous) | + +Optional params: + +| Parameter | Description | +|----------------------|--------------------------------------------------| +| require_crendentials | true/false. flag to require registry credentials | + +Required secrets: + +| Secret Name | Details | +|-------------|---------------------| +| dockeruser | the docker user | +| dockerpass | the docker password | +| dockeremail | the docker email | + +#### command + +Runs a command on the newly create docker image or on the image specified. + +Required params: + +| Parameter | Description | +|-----------|------------------------------------------------| +| command | list of string defining the command to execute | + +Optional params: + +| Parameter | Description | +|--------------|--------------------------------------------------------------| +| args | list of string defining the args for the command | +| image | custom image to use for running the build | +| dependencies | list of dependencies to run, these are kubernetes spec files | +| working_dir | change the working directory | + +#### deploy + +Deploys a kubernetes spec. + +Required params: + +| Parameter | Description | +|-------------|------------------------------------| +| deploy_file | the kubernetes spec file to deploy | + diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 0000000..21bf10d --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,78 @@ +Kontinuous Setup +================ + +This document details the setup of Kontinuous without using the CLI bootstrap. + +## Dependencies + +Kontinuous is dependent of the following: + +### Kubernetes + +Kontinuous should runs on top of a Kubernetes cluster. It uses Jobs heavily so it will require at least v1.1 with Jobs enabled. + +### etcd + +etcd is used as a backend for storing pipeline and build details. This is a dedicated instance to avoid polluting the Kubernetes etcd cluster. + +### Minio + +Minio is used to store logs and artifacts. S3 could also be used as it is compatible with minio although this hasn't been tested yet. + +### Docker Registry + +Kontinuous stores docker registry internal and uses an internal docker registry. + +## Running in Kubernetes + +Kontinuous is meant to run inside a kubernetes cluster, preferrably by a Deployment or Replication Controller. + +### Docker Image + +The docker image can be found here: [quay.io/acaleph/kontinuous](https://quay.io/acaleph/kontinuous) + +### Environment Variables + +The following environment variables needs to be defined: + +| Environment Variable | Description | Example | +|----------------------|-----------------------------------------|------------------------| +| KV_ADDRESS | The etcd address | etcd:2379 | +| S3_URL | The minio address | http://minio:9000 | +| KONTINUOUS_URL | The address where kontinuous is running | http://kontinuous:3005 | +| INTERNAL_REGISTRY | The internal registry address | internal-registry:5000 | + +### Secrets + +A Kubernetes Secret needs to be defined and mounted on `/.secret`. The secret should have a key named `kontinuous-secrets` and contains the following data (must be base64 encoded): + +```json +{ + "AuthSecret": "base64 encoded auth secret", + "S3SecretKey": "s3 secret key", + "S3AccessKey": "s3 access key", + "GithubClientID": "github client ID", + "GithubClientSecret": "github client secret" +} +``` + +#### AuthSecret + +AuthSecret is the secret used for signing JSON Web Tokens used for authentication. This can be any base64 encoded string. More details about Authentication can be found [here](docs/api.md). + +#### S3AccessKey & S3SecretKey + +S3AccessKey and S3SecretKey are the keys taken from Minio. These can be retrieved from minio using the following command: + +``` +$ kubectl logs --namespace={namespace} {minio-pod-name} +``` + +#### GithubClientID & GithubClientSecret + +GithubClientID and GithubClientSecret are optional. They are needed if running Kontinuous UI as the UI requires Github login. These are taken from the Github OAuth Application details. More details about Authentication can be found [here](docs/api.md) + +### Ports + +Kontinuous uses port `3005`. This needs to be exposed. + diff --git a/k8s-spec.yml.example.yaml b/k8s-spec.yml.example similarity index 100% rename from k8s-spec.yml.example.yaml rename to k8s-spec.yml.example From 289fed568296a0554aa98cff6242caba835a2b51 Mon Sep 17 00:00:00 2001 From: dexter Date: Thu, 19 May 2016 16:48:27 +0800 Subject: [PATCH 19/30] added pipeline example in README --- README.md | 36 +++++++++++++++++++++++++++++++++++- docs/pipeline.md | 5 +---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3f36180..2d8a578 100644 --- a/README.md +++ b/README.md @@ -53,12 +53,46 @@ Once a public IP for `kontinuous-ui` is available, the Github OAuth Application Alternatively, for more customization, a sample yaml file for running kontinuous and its dependencies in Kubernetes can be found [here](./k8s-spec.yml.example). More details can be found [here](docs/setup.md). -Once running, add a [.pipeline.yml](#pipeline-spec) to the root of your Github repo and configure the webhooks. More details about pipeline spec creation can be found [here](docs/pipeline.md). +Once running, add a [.pipeline.yml](#pipeline-spec) to the root of your Github repo and configure the webhooks. Example pipelines can be found in [/examples](./examples) The [CLI client](#clients) or [API](#api) can be used to view build status or logs. +## Pipeline Specification + +Pipeline specification should be at the root directory of the repository. This defines the stages of the builds. More details about pipeline spec creation can be found [here](docs/pipeline.md). + +```yaml +--- +kind: Pipeline +apiVersion: v1alpha1 +metadata: + name: kontinuous + namespace: acaleph +spec: + selector: + matchLabels: + app: kontinuous + type: ci-cd + template: + metadata: + name: kontinuous + labels: + app: kontinuous + type: ci-cd + notif: + - type: slack + secrets: + - notifcreds + - docker-credentials + stages: + - name: Build Docker Image + type: docker_build +``` + +This example only has one stage, build a docker image. + ## Clients There are two clients currently available: diff --git a/docs/pipeline.md b/docs/pipeline.md index 1307660..edc6d58 100644 --- a/docs/pipeline.md +++ b/docs/pipeline.md @@ -45,9 +45,6 @@ spec: external_registry: quay.io external_image_name: acaleph/kontinuous require_credentials: "TRUE" - username: user # taken from secret - password: password # taken from secret - email: email # taken from secret ``` Some important fields on the spec: @@ -136,7 +133,7 @@ Optional params: | Parameter | Description | |----------------------|--------------------------------------------------| -| require_crendentials | true/false. flag to require registry credentials | +| require_crendentials | TRUE/FALSE. flag to require registry credentials | Required secrets: From 3ead78e2b82a5f86964bf84fa65e41c22c153e36 Mon Sep 17 00:00:00 2001 From: madz Date: Fri, 20 May 2016 15:13:52 +0800 Subject: [PATCH 20/30] minio latest change on paramater insecure to secure. requires us make changes on the value. --- store/mc/mc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/mc/mc.go b/store/mc/mc.go index a2bd66a..c132d99 100644 --- a/store/mc/mc.go +++ b/store/mc/mc.go @@ -20,7 +20,7 @@ func NewMinioClient(host, accessKey, secretKey string) (*MinioClient, error) { fqdn = h[0] } - client, err := minio.NewV4(fqdn, accessKey, secretKey, true) + client, err := minio.NewV4(fqdn, accessKey, secretKey, false) if err != nil { return nil, err } From 2f01a3e90e786b248ddca86d9246379ab4e25735 Mon Sep 17 00:00:00 2001 From: Hunter Nield Date: Fri, 20 May 2016 20:43:30 +0800 Subject: [PATCH 21/30] Deployment and `apply` resources --- cli/deploy.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/cli/deploy.go b/cli/deploy.go index 1b7e3b1..dcf7962 100644 --- a/cli/deploy.go +++ b/cli/deploy.go @@ -41,7 +41,7 @@ metadata: namespace: {{.Namespace}} data: kontinuous-secrets: {{.SecretData}} - + ` var minioSvc = ` @@ -65,8 +65,8 @@ spec: ` var minioRc = ` --- -kind: ReplicationController -apiVersion: v1 +apiVersion: extensions/v1beta1 +kind: Deployment metadata: name: minio namespace: {{.Namespace}} @@ -128,8 +128,8 @@ spec: var etcdRc = ` --- -kind: ReplicationController -apiVersion: v1 +apiVersion: extensions/v1beta1 +kind: Deployment metadata: name: etcd namespace: {{.Namespace}} @@ -183,8 +183,8 @@ spec: var registryRc = ` --- -kind: ReplicationController -apiVersion: v1 +apiVersion: extensions/v1beta1 +kind: Deployment metadata: name: registry namespace: {{.Namespace}} @@ -236,8 +236,8 @@ spec: var kontinuousRc = ` --- -kind: ReplicationController -apiVersion: v1 +apiVersion: extensions/v1beta1 +kind: Deployment metadata: name: kontinuous namespace: {{.Namespace}} @@ -306,8 +306,8 @@ spec: var dashboardRc = ` --- -apiVersion: v1 -kind: ReplicationController +apiVersion: extensions/v1beta1 +kind: Deployment metadata: labels: app: kontinuous-ui @@ -411,7 +411,7 @@ func encryptSecret(secret string) string { } func createKontinuousResouces(path string) error { - cmd := fmt.Sprintf("kubectl create -f %s", path) + cmd := fmt.Sprintf("kubectl apply -f %s", path) _, err := exec.Command("bash", "-c", cmd).Output() if err != nil { return err @@ -440,6 +440,7 @@ func deleteKontinuousResources() error { func fetchKontinuousIP(serviceName, namespace string) (string, error) { var ip string + // TODO: test out {{range .status.loadBalancer.ingress }}{{.ip}}{{end}} cmd := fmt.Sprintf(`kubectl get svc %s --namespace=%s -o template --template="{{.status.loadBalancer.ingress}}"`, serviceName, namespace) for len(ip) == 0 { out, err := exec.Command("bash", "-c", cmd).Output() @@ -473,7 +474,7 @@ func deployResource(definition string, fileName string, deploy *Deploy) error { cmd = fmt.Sprintf("kubectl get -f %s", filePath) _, err := exec.Command("bash", "-c", cmd).Output() if err == nil { - return errors.New(fmt.Sprintf("%s: %s already exists \n", resourceType, resourceName)) + fmt.Sprintf("%s: %s already exists, applying resource again... \n", resourceType, resourceName) } err = createKontinuousResouces(filePath) @@ -566,12 +567,12 @@ func DeployKontinuous(namespace, authcode, clientid, clientsecret string) error err = deployResource(kontinuousRc, "KONTINUOUS_RC", &deploy) if err != nil { - fmt.Println("Unable to deploy Kontinuous Deploy Replication Controller \n %s", err.Error()) + fmt.Println("Unable to deploy Kontinuous Deployment \n %s", err.Error()) return err } err = deployResource(dashboardRc, "DASHBOARD_RC", &deploy) if err != nil { - fmt.Printf("Unable to deploy Kontinuous UI Replication Controller \n %s", err.Error()) + fmt.Printf("Unable to deploy Kontinuous UI Deployment \n %s", err.Error()) return err } From 4ca1210dae0ea01251b041267940b82236047839 Mon Sep 17 00:00:00 2001 From: madz Date: Mon, 23 May 2016 10:29:52 +0800 Subject: [PATCH 22/30] add matchLabels to deployment selector. Change dockerpass to dockerpassword in README.md --- cli/deploy.go | 45 +++++++++++++++++++++++++-------------------- docs/pipeline.md | 10 +++++----- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/cli/deploy.go b/cli/deploy.go index dcf7962..71e13de 100644 --- a/cli/deploy.go +++ b/cli/deploy.go @@ -63,7 +63,7 @@ spec: port: 9000 targetPort: 9000 ` -var minioRc = ` +var minioDep = ` --- apiVersion: extensions/v1beta1 kind: Deployment @@ -76,8 +76,9 @@ metadata: spec: replicas: 1 selector: - app: minio - type: object-store + matchLabels: + app: minio + type: object-store template: metadata: name: minio @@ -126,7 +127,7 @@ spec: targetPort: 2379 ` -var etcdRc = ` +var etcdDep = ` --- apiVersion: extensions/v1beta1 kind: Deployment @@ -139,8 +140,9 @@ metadata: spec: replicas: 1 selector: - app: etcd - type: kv + matchLabels: + app: etcd + type: kv template: metadata: labels: @@ -181,7 +183,7 @@ spec: targetPort: 5000 ` -var registryRc = ` +var registryDep = ` --- apiVersion: extensions/v1beta1 kind: Deployment @@ -194,8 +196,9 @@ metadata: spec: replicas: 1 selector: - app: registry - type: storage + matchLabels: + app: registry + type: storage template: metadata: name: registry @@ -234,7 +237,7 @@ spec: targetPort: 3005 ` -var kontinuousRc = ` +var kontinuousDep = ` --- apiVersion: extensions/v1beta1 kind: Deployment @@ -247,8 +250,9 @@ metadata: spec: replicas: 1 selector: - app: kontinuous - type: ci-cd + matchLabels: + app: kontinuous + type: ci-cd template: metadata: labels: @@ -304,7 +308,7 @@ spec: ` -var dashboardRc = ` +var dashboardDep = ` --- apiVersion: extensions/v1beta1 kind: Deployment @@ -317,8 +321,9 @@ metadata: spec: replicas: 1 selector: - app: kontinuous-ui - type: dashboard + matchLabels: + app: kontinuous-ui + type: dashboard template: metadata: labels: @@ -542,11 +547,11 @@ func DeployKontinuous(namespace, authcode, clientid, clientsecret string) error deployResource(namespaceData, "APP_NAMESPACE", &deploy) deployResource(minioSvc, "MINIO_SVC", &deploy) - deployResource(minioRc, "MINIO_RC", &deploy) + deployResource(minioDep, "MINIO_DEPLOYMENT", &deploy) deployResource(etcdSvc, "ETCD_SVC", &deploy) - deployResource(etcdRc, "ETCD_RC", &deploy) + deployResource(etcdDep, "ETCD_DEPLOYMENT", &deploy) deployResource(registrySvc, "REGISTRY_SVC", &deploy) - deployResource(registryRc, "REGISTRY_RC", &deploy) + deployResource(registryDep, "REGISTRY_DEPLOYMENT", &deploy) deployResource(kontinuousSvc, "KONTINUOUS_SVC", &deploy) deployResource(dashboardSvc, "DASHBOARD_SVC", &deploy) @@ -564,13 +569,13 @@ func DeployKontinuous(namespace, authcode, clientid, clientsecret string) error deploy.DashboardIP = dashboardIp deploy.KontinuousIP = ip - err = deployResource(kontinuousRc, "KONTINUOUS_RC", &deploy) + err = deployResource(kontinuousDep, "KONTINUOUS_DEPLOYMENT", &deploy) if err != nil { fmt.Println("Unable to deploy Kontinuous Deployment \n %s", err.Error()) return err } - err = deployResource(dashboardRc, "DASHBOARD_RC", &deploy) + err = deployResource(dashboardDep, "DASHBOARD_DEPLOYMENT", &deploy) if err != nil { fmt.Printf("Unable to deploy Kontinuous UI Deployment \n %s", err.Error()) return err diff --git a/docs/pipeline.md b/docs/pipeline.md index edc6d58..a31d542 100644 --- a/docs/pipeline.md +++ b/docs/pipeline.md @@ -137,11 +137,11 @@ Optional params: Required secrets: -| Secret Name | Details | -|-------------|---------------------| -| dockeruser | the docker user | -| dockerpass | the docker password | -| dockeremail | the docker email | +| Secret Name | Details | +|-----------------|---------------------| +| dockeruser | the docker user | +| dockerpassword | the docker password | +| dockeremail | the docker email | #### command From cfcd2c6f70f57c2e449706fa34ef4ecde8250bec Mon Sep 17 00:00:00 2001 From: Hunter Nield Date: Mon, 23 May 2016 11:32:31 +0800 Subject: [PATCH 23/30] Use alpine:edge to avoid DNS issues --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8528d37..d10a071 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.6-alpine +FROM alpine:edge ENV GOPATH /go ENV SWAGGER_UI /swagger/dist @@ -10,12 +10,12 @@ RUN mkdir /swagger && tar xvzf third_party/swagger.tar.gz -C /swagger # create and remove downloaded libraries RUN apk update && \ - apk add make git && \ + apk add make git go ca-certificates && \ make && \ mv build/bin/kontinuous /bin && \ mv build/bin/kontinuous-cli /bin && \ rm -rf /go && \ - apk del --purge make git && \ + apk del --purge make git go && \ rm -rf /var/cache/apk/* EXPOSE 3005 From 3d4fa6c29a0df36108ca2fea04c2c9a9174f216d Mon Sep 17 00:00:00 2001 From: madz Date: Mon, 23 May 2016 11:46:43 +0800 Subject: [PATCH 24/30] add namespace to service to enable other jobs it from different namespace --- cli/deploy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/deploy.go b/cli/deploy.go index 71e13de..4df602e 100644 --- a/cli/deploy.go +++ b/cli/deploy.go @@ -271,7 +271,7 @@ spec: - name: KV_ADDRESS value: etcd:2379 - name: S3_URL - value: http://minio:9000 + value: http://minio.{{.Namespace}}:9000 - name: KONTINUOUS_URL value: http://{{.KontinuousIP}}:8080 - name: INTERNAL_REGISTRY From 8debc5a6ca824c69104a4879fe47993dbe38e4c1 Mon Sep 17 00:00:00 2001 From: madz Date: Mon, 23 May 2016 12:06:26 +0800 Subject: [PATCH 25/30] add namespace to service to enable other jobs it from different namespace --- cli/deploy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/deploy.go b/cli/deploy.go index 4df602e..edb338a 100644 --- a/cli/deploy.go +++ b/cli/deploy.go @@ -275,7 +275,7 @@ spec: - name: KONTINUOUS_URL value: http://{{.KontinuousIP}}:8080 - name: INTERNAL_REGISTRY - value: registry:5000 + value: registry.{{.Namespace}}:5000 ports: - name: api containerPort: 3005 From 0cd8a413a3b4f47448f4aa8c89528f7cc84286e2 Mon Sep 17 00:00:00 2001 From: madz Date: Mon, 23 May 2016 16:03:19 +0800 Subject: [PATCH 26/30] use clusterIP instead of dns for registry --- cli/deploy.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/cli/deploy.go b/cli/deploy.go index edb338a..1b9f79a 100644 --- a/cli/deploy.go +++ b/cli/deploy.go @@ -275,7 +275,7 @@ spec: - name: KONTINUOUS_URL value: http://{{.KontinuousIP}}:8080 - name: INTERNAL_REGISTRY - value: registry.{{.Namespace}}:5000 + value: {{.RegistryIP}}:5000 ports: - name: api containerPort: 3005 @@ -357,6 +357,7 @@ type Deploy struct { SecretData string KontinuousIP string DashboardIP string + RegistryIP string GHClient string GHSecret string } @@ -464,6 +465,27 @@ func fetchKontinuousIP(serviceName, namespace string) (string, error) { return ip, nil } +func fetchClusterIP(serviceName, namespace string) (string, error) { + var ip string + + // TODO: test out {{range .status.loadBalancer.ingress }}{{.ip}}{{end}} + cmd := fmt.Sprintf(`kubectl get svc %s --namespace=%s -o template --template="{{.spec.clusterIP}}"`, serviceName, namespace) + for len(ip) == 0 { + out, err := exec.Command("bash", "-c", cmd).Output() + if err != nil { + return "", err + } + + outStr := string(out) + if !strings.Contains(outStr, "") && !strings.Contains(outStr, "") { + ip = strings.TrimSpace(outStr) + } else { + time.Sleep(5 * time.Second) + } + } + return ip, nil +} + func deployResource(definition string, fileName string, deploy *Deploy) error { resource, _ := generateResource(definition, deploy) resourceArr := strings.Split(fileName, "_") @@ -566,8 +588,11 @@ func DeployKontinuous(namespace, authcode, clientid, clientsecret string) error ip, _ := fetchKontinuousIP("kontinuous", deploy.Namespace) dashboardIp, _ := fetchKontinuousIP("kontinuous-ui", deploy.Namespace) + registryIp, _ := fetchClusterIP("registry", deploy.Namespace) + deploy.DashboardIP = dashboardIp deploy.KontinuousIP = ip + deploy.RegistryIP = registryIp err = deployResource(kontinuousDep, "KONTINUOUS_DEPLOYMENT", &deploy) From bcd30b61e202cc4695266040270dc8f168351c6c Mon Sep 17 00:00:00 2001 From: madz Date: Mon, 23 May 2016 16:22:03 +0800 Subject: [PATCH 27/30] retain make for kontinuous unit test --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d10a071..519b746 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN apk update && \ mv build/bin/kontinuous /bin && \ mv build/bin/kontinuous-cli /bin && \ rm -rf /go && \ - apk del --purge make git go && \ + apk del --purge git go && \ rm -rf /var/cache/apk/* EXPOSE 3005 From 86be467263184cd687faf77e8dc65821e41b7b91 Mon Sep 17 00:00:00 2001 From: madz Date: Mon, 23 May 2016 16:30:31 +0800 Subject: [PATCH 28/30] retain go for kontinuous unit test --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 519b746..6ac64f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN apk update && \ mv build/bin/kontinuous /bin && \ mv build/bin/kontinuous-cli /bin && \ rm -rf /go && \ - apk del --purge git go && \ + apk del --purge git && \ rm -rf /var/cache/apk/* EXPOSE 3005 From 98e8880e202c0f89c66e9e6da5bb4f3645c456fa Mon Sep 17 00:00:00 2001 From: madz Date: Mon, 23 May 2016 16:44:15 +0800 Subject: [PATCH 29/30] retain code for kontinuous unit test --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6ac64f1..4f7b250 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,6 @@ RUN apk update && \ make && \ mv build/bin/kontinuous /bin && \ mv build/bin/kontinuous-cli /bin && \ - rm -rf /go && \ apk del --purge git && \ rm -rf /var/cache/apk/* From 0a836bfe9082e3664251631cfd9ba1b983d61d35 Mon Sep 17 00:00:00 2001 From: madz Date: Mon, 23 May 2016 18:16:45 +0800 Subject: [PATCH 30/30] Updated kontinuous documentation --- README.md | 2 +- cli/README.md | 25 ++++++++++++++ docs/setup.md | 12 ++++++- k8s-spec.yml.example | 81 +++++++++++++++++++++++++++++++++----------- 4 files changed, 99 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 2d8a578..beef290 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Before running Kontinuous, it needs to be added as a github OAuth Application [h The `kubernetes-cli` can bootstrap a kontinuous setup on a running Kubernetes cluster. This requires `kubectl` to be in the `PATH` and configured to access the cluster. ``` -$ kotinuous-cli --namespace {namespace} \ +$ kontinuous-cli --namespace {namespace} \ --auth-secret {base64 encoded secret} \ --github-client-id {github client id} \ --github-client-secret {github client secret} diff --git a/cli/README.md b/cli/README.md index 8b31f4e..677b87d 100644 --- a/cli/README.md +++ b/cli/README.md @@ -27,8 +27,33 @@ Example: getting a list of pipelines. $ kontinuous-cli get-pipelines ``` +Deploy Kontinuous to the cluster + +``` +$ kontinuous-cli --namespace {namespace} \ + --auth-secret {base64 encoded secret} \ + --github-client-id {github client id} \ + --github-client-secret {github client secret} +``` + + +Remove recently deployed Kontinuous from the cluster + +``` +$ kontinuous-cli deploy remove +``` + ## Notes +Kontinuous internal registry uses Cluster IP. Should there be any changes on the IP address, please execute the following cli command: + +``` +$ kontinuous-cli --namespace {namespace} \ + --auth-secret {base64 encoded secret} \ + --github-client-id {github client id} \ + --github-client-secret {github client secret} +``` + This is still WIP. diff --git a/docs/setup.md b/docs/setup.md index 21bf10d..ac0d31b 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -39,7 +39,7 @@ The following environment variables needs to be defined: |----------------------|-----------------------------------------|------------------------| | KV_ADDRESS | The etcd address | etcd:2379 | | S3_URL | The minio address | http://minio:9000 | -| KONTINUOUS_URL | The address where kontinuous is running | http://kontinuous:3005 | +| KONTINUOUS_URL | The address where kontinuous is running | http://kontinuous:8080 | | INTERNAL_REGISTRY | The internal registry address | internal-registry:5000 | ### Secrets @@ -76,3 +76,13 @@ GithubClientID and GithubClientSecret are optional. They are needed if running K Kontinuous uses port `3005`. This needs to be exposed. + +## Notes + +Kontinuous internal registry uses Cluster IP. Should there be any changes on the IP address, please execute the following command: + +``` +kubectl apply -f < KONTINUOUS_SPEC_FILE.yml > +``` + + diff --git a/k8s-spec.yml.example b/k8s-spec.yml.example index 7535aeb..6d83487 100644 --- a/k8s-spec.yml.example +++ b/k8s-spec.yml.example @@ -7,6 +7,59 @@ metadata: data: kontinuous-secrets: {base64 encoded secrets} +--- +kind: Service +apiVersion: v1 +metadata: + name: etcd + namespace: acaleph + labels: + app: etcd + type: kv +spec: + selector: + app: etcd + type: kv + ports: + - name: api + port: 2379 + targetPort: 2379 + + +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: etcd + namespace: acaleph + labels: + app: etcd + type: kv +spec: + replicas: 1 + selector: + matchLabels: + app: etcd + type: kv + template: + metadata: + labels: + app: etcd + type: kv + spec: + containers: + - name: etcd + image: quay.io/coreos/etcd:v2.2.2 + imagePullPolicy: Always + args: + - --listen-client-urls + - http://0.0.0.0:2379 + - --advertise-client-urls + - http://0.0.0.0:2379 + ports: + - name: api + containerPort: 2379 + --- kind: Service apiVersion: v1 @@ -27,8 +80,8 @@ spec: targetPort: 3005 --- -kind: ReplicationController -apiVersion: v1 +kind: Deployment +apiVersion: extensions/v1beta1 metadata: name: kontinuous namespace: acaleph @@ -38,8 +91,9 @@ metadata: spec: replicas: 1 selector: - app: kontinuous - type: ci-cd + matchLabels: + app: kontinuous + type: ci-cd template: metadata: labels: @@ -51,17 +105,6 @@ spec: secret: secretName: kontinuous-secrets containers: - - name: etcd - image: quay.io/coreos/etcd:v2.2.2 - imagePullPolicy: Always - args: - - --listen-client-urls - - http://0.0.0.0:2379 - - --advertise-client-urls - - http://0.0.0.0:2379 - ports: - - name: api - containerPort: 2379 - name: acaleph-deploy image: quay.io/acaleph/kontinuous:latest imagePullPolicy: Always @@ -70,13 +113,13 @@ spec: containerPort: 3005 env: - name: KV_ADDRESS - value: localhost:2379 + value: etcd:2379 - name: S3_URL - value: http://minio:9000 + value: http://minio.acaleph:9000 - name: KONTINUOUS_URL - value: http://kontinuous:8080 + value: http://{{kontinuous}}:8080 - name: INTERNAL_REGISTRY - value: internal-registry:5000 + value: {{internal-registry}}:5000 volumeMounts: - name: kontinuous-secrets mountPath: /.secret