{Earthfile, Jenkinsfile}: Earthly ci testing

Change-Id: I6c80ed5468fc7c89ce3d99568700382b1f42fa51
storj · Oct 12, 2023 · afac8ec · afac8ec
1 parent 7074e31
commit afac8ec
Show file tree

Hide file tree

Showing 12 changed files with 173 additions and 137 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -8,11 +8,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Setup Earthly
-        run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.6.23/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'"
+        run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.7.20/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'"
       - name: Test
         run: earthly --ci +test
       - name: Lint
         run: earthly --ci +lint
       - name: Integration
-        run: earthly -P --ci +integration
+        run: earthly -P --ci +integration-all
 
diff --git a/Earthfile b/Earthfile
@@ -1,12 +1,14 @@
-VERSION 0.6
+VERSION 0.7
 FROM golang:1.19
 WORKDIR /go/storj-up
 
-lint:
-    WORKDIR /go/storj-up
+lint-deps:
     RUN go install github.com/storj/ci/...@5286f52
     RUN go install honnef.co/go/tools/cmd/[email protected]
     RUN go install github.com/golangci/golangci-lint/cmd/[email protected]
+
+lint:
+    FROM +lint-deps
     COPY . /go/storj-up
     RUN staticcheck ./...
     RUN golangci-lint --build-tags mage -j=2 run
@@ -18,7 +20,19 @@ lint:
     RUN check-monkit ./...
     RUN check-mod-tidy
 
-build:
+build-app-deps:
+    # Download deps before copying code.
+    COPY go.mod go.sum ./pkg/config/gen/go.mod ./pkg/config/gen/go.sum .
+    RUN go mod download
+    # Output these back in case go mod download changes them.
+    SAVE ARTIFACT go.mod AS LOCAL go.mod
+    SAVE ARTIFACT go.sum AS LOCAL go.sum
+    SAVE ARTIFACT ./pkg/config/gen/go.mod AS LOCAL ./pkg/config/gen/go.mod
+    SAVE ARTIFACT ./pkg/config/gen/go.sum AS LOCAL ./pkg/config/gen/go.sum
+
+build-app:
+    FROM +build-app-deps
+    # Copy and build code.
     COPY . .
     RUN --mount=type=cache,target=/root/.cache/go-build \
         --mount=type=cache,target=/go/pkg/mod \
@@ -40,22 +54,58 @@ test:
        ./scripts/test.sh
    SAVE ARTIFACT build/tests.json AS LOCAL build/tests.json
 
-integration:
-   FROM earthly/dind:ubuntu
-   RUN apt-get update && apt-get install -y golang-go docker-compose-plugin gcc
-   ENV PATH=$PATH:/root/go/bin
-   COPY . .
-   WORKDIR /test
-   WITH DOCKER --pull img.dev.storj.io/storjup/storj:1.83.2 \
-               --pull img.dev.storj.io/storjup/edge:1.58.1 \
-               --pull cockroachdb/cockroach \
-               --pull redis:6.0.9 \
-               --pull img.dev.storj.io/storjup/storjscan:1.0.3 \
-               --pull ethereum/client-go
-      RUN ./test.sh && \
-          ./test-edge.sh && \
-          ./test-storjscan.sh
-   END
+integration-all:
+   BUILD +integration-uplink
+   BUILD +integration-edge
+   BUILD +integration-storjscan
+
+integration-uplink:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build-app/storj-up .
+    RUN /go/storj-up/storj-up init minimal,db,uplink
+    COPY ./test/uplink/basic_upload_download.sh basic_upload_download.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 uplink bash -c 'storj-up health --host cockroach -d 90' && \
+            docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -e >> ~/.bashrc' && \
+            < basic_upload_download.sh docker compose exec -T -u 0 uplink bash -i
+    END
+
+integration-edge:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build-app/storj-up .
+    RUN /go/storj-up/storj-up init minimal,edge,db,uplink && \
+        /go/storj-up/storj-up image uplink img.dev.storj.io/dlm/storj:1 && \
+        /go/storj-up/storj-up env set authservice STORJ_ENDPOINT=http://gateway-mt:9999
+    COPY ./test/edge/basic_rclone.sh basic_rclone.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 uplink bash -c 'storj-up health --host cockroach -d 90' && \
+            docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e --s3 >> ~/.bashrc' && \
+            < basic_rclone.sh docker compose exec -T -u 0 uplink bash -i
+    END
+
+integration-storjscan:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build-app/storj-up .
+    RUN /go/storj-up/storj-up init && \
+        /go/storj-up/storj-up init minimal,satellite-core,edge,db,billing && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_BILLING_CONFIG_INTERVAL=5s && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_INTERVAL=5s && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_CONFIRMATIONS=12 && \
+        /go/storj-up/storj-up env setenv storjscan STORJ_TOKEN_PRICE_USE_TEST_PRICES=true && \
+        /go/storj-up/storj-up image storjscan img.dev.storj.io/dlm/storjscan:1
+    COPY ./test/storjscan/basic_storjscan.sh basic_storjscan.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 storjscan bash -c 'storj-up health --host cockroach -d 90' && \
+            docker compose exec -T -u 0 storjscan bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e >> ~/.bashrc' && \
+            docker compose cp pk.json storjscan:/var/lib/storj/pk.json && \
+            docker compose cp pass storjscan:/var/lib/storj/pass && \
+            docker compose cp TestToken.abi storjscan:/var/lib/storj/TestToken.abi && \
+            docker compose cp TestToken.bin storjscan:/var/lib/storj/TestToken.bin && \
+            < basic_storjscan.sh docker compose exec -T -u 0 storjscan bash -i
+    END
 
 check-format:
    COPY . .

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,6 +1,6 @@
 pipeline {
     agent {
-       label 'node4'
+       label('nomad || node4')
     }
 
     options {
@@ -17,20 +17,43 @@ pipeline {
                checkout scm
             }
         }
-	stage('Lint') {
-		steps {
-			sh "earthly +lint"
-		}
-	}
-	stage('Test') {
-		steps {
-			sh "earthly +test"
-		}
-	}
-	stage('Integration') {
-		steps {
-			sh "earthly -P +integration"
-		}
-	}
+        stage('Lint') {
+            steps {
+                sh "earthly +lint"
+            }
+        }
+        stage('Test') {
+            steps {
+                sh "earthly +test"
+            }
+        }
+        stage('Integration') {
+            parallel {
+                stage('Uplink') {
+                    steps {
+                        withCredentials([usernamePassword(credentialsId: '67d52f80-1802-407a-a21a-ea24d4c28a83', passwordVariable: 'password', usernameVariable: 'username')]) {
+                            sh 'docker login img.dev.storj.io -u $username -p $password'
+                        }
+                        sh "earthly -P +integration-uplink"
+                    }
+                }
+                stage('Edge') {
+                    steps {
+                        withCredentials([usernamePassword(credentialsId: '67d52f80-1802-407a-a21a-ea24d4c28a83', passwordVariable: 'password', usernameVariable: 'username')]) {
+                            sh 'docker login img.dev.storj.io -u $username -p $password'
+                        }
+                        sh "earthly -P +integration-edge"
+                    }
+                }
+                stage('Storjscan') {
+                    steps {
+                        withCredentials([usernamePassword(credentialsId: '67d52f80-1802-407a-a21a-ea24d4c28a83', passwordVariable: 'password', usernameVariable: 'username')]) {
+                            sh 'docker login img.dev.storj.io -u $username -p $password'
+                        }
+                        sh "earthly -P +integration-storjscan"
+                    }
+                }
+            }
+        }
     }
 }