Merge pull request #270 from asfadmin/test

Release v1.2.0
asfadmin · Jul 11, 2023 · efc7f96 · efc7f96
2 parents 78a78c4 + ed60ac7
commit efc7f96
Show file tree

Hide file tree

Showing 30 changed files with 87 additions and 47 deletions.
diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
@@ -14,6 +14,6 @@ on:
 jobs:
   call-changelog-check-workflow:
     # Docs: https://github.com/ASFHyP3/actions
-    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.7.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.8.0
     secrets:
       USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml
@@ -13,4 +13,4 @@ on:
 jobs:
   call-labeled-pr-check-workflow:
     # Docs: https://github.com/ASFHyP3/actions
-    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.7.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.8.0
diff --git a/.github/workflows/release-checklist-comment.yml b/.github/workflows/release-checklist-comment.yml
@@ -10,7 +10,7 @@ on:
 jobs:
   call-release-workflow:
     # Docs: https://github.com/ASFHyP3/actions
-    uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.7.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.8.0
     permissions:
       pull-requests: write
     secrets:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.7.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.8.0
     with:
       release_prefix: GRFN Ingest
       release_branch: prod

diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml
@@ -8,7 +8,7 @@ on:
 jobs:
   call-bump-version-workflow:
     # Docs: https://github.com/ASFHyP3/actions
-    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.7.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.8.0
     with:
       user: tools-bot
       email: [email protected]

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.2.0]
+### Removed
+- ` (BETA)` from collection data set ID values to match new collection titles in grfn-cmr v2.1.0
+
 ## [1.1.0]
 ### Added
 - `VERSION` attribute is now populated in CMR and `version` is now a required field in the input metadata file

diff --git a/README.md b/README.md
@@ -20,11 +20,6 @@ data.
 * **echo10-to-cmr:** A scheduled Lambda function that submits ECHO 10 XML metadata files to CMR.
 * **notify:** A Lambda function that sends ingest success/failure messages to the SNS response topic.
 
-# Build and Deployment Instructions
-
-1. Run steps in buildspec.yaml
-2. Deploy cloudformation-final.yaml
-
 # Top Level Inputs and Outputs
 
 ## Runtime Inputs
@@ -48,6 +43,27 @@ Runtime inputs consist of the following staged files:
 * Sentinel-1 All Interferometric Products (BETA) \<product-name\>-All
 * Success/failure message sent to DefaultResponseTopicArn
 
+# Integration testing
+
+From the repository root, run:
+
+```bash
+aws --profile grfn sns publish --topic-arn "arn:aws:sns:us-east-1:406893895021:ingest-test-jobs" --message file://tests/example-message.json
+```
+
+Wait a few minutes for the job to process (you can monitor it in the `ingest-test-jobs` Step Function).
+
+There should be five CMR products published with each job:
+the base NetCDF file, plus four GeoTIFFs: amplitude, coherence, unwrapped phase, and connected components.
+
+Review the published products in CMR UAT using the following link:
+<https://cmr.uat.earthdata.nasa.gov/search/granules.echo10?provider=ASF&producer_granule_id=S1-GUNW-A-R-014-tops-20230116_20220214-153036-00038E_00022N-PP-3708-v2_0_6>
+
+You can also replace `.echo10` with `.json` or `.umm_json`.
+
+Re-running a new job with the same `ProductName` field will overwrite the existing records in CMR.
+Re-running the exact same job should only change the `InsertTime` and `LastUpdate` fields.
+
 # Credits
 
 ## ASF

diff --git a/echo10-construction/cloudformation.yaml b/echo10-construction/cloudformation.yaml
@@ -89,7 +89,7 @@ Resources:
                   {"name": "S1 I-grams (BETA) - Other", "coords": [[-180,-90],[180,-90],[180,90],[-180,90]]}
                 ],
                 "collection": {
-                  "dataset_id": "Sentinel-1 Interferograms (BETA)",
+                  "dataset_id": "Sentinel-1 Interferograms",
                   "processing_type": "GUNW_STD",
                   "processing_type_display": "Standard Product, NetCDF",
                   "processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
@@ -99,31 +99,31 @@ Resources:
                 {
                   "label": "unwrappedPhase",
                   "layer": "science/grids/data/unwrappedPhase",
-                  "dataset_id": "Sentinel-1 Interferograms - Unwrapped Phase (BETA)",
+                  "dataset_id": "Sentinel-1 Interferograms - Unwrapped Phase",
                   "processing_type": "GUNW_UNW",
                   "processing_type_display": "Unwrapped Phase, GeoTIFF",
                   "processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
                 },
                 {
                   "label": "coherence",
                   "layer": "science/grids/data/coherence",
-                  "dataset_id": "Sentinel-1 Interferograms - Coherence (BETA)",
+                  "dataset_id": "Sentinel-1 Interferograms - Coherence",
                   "processing_type": "GUNW_COH",
                   "processing_type_display": "Coherence, GeoTIFF",
                   "processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
                 },
                 {
                   "label": "amplitude",
                   "layer": "science/grids/data/amplitude",
-                  "dataset_id": "Sentinel-1 Interferograms - Amplitude (BETA)",
+                  "dataset_id": "Sentinel-1 Interferograms - Amplitude",
                   "processing_type": "GUNW_AMP",
                   "processing_type_display": "Amplitude, GeoTIFF",
                   "processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
                 },
                 {
                   "label": "connectedComponents",
                   "layer": "science/grids/data/connectedComponents",
-                  "dataset_id": "Sentinel-1 Interferograms - Connected Components (BETA)",
+                  "dataset_id": "Sentinel-1 Interferograms - Connected Components",
                   "processing_type": "GUNW_CON",
                   "processing_type_display": "Connected Components, GeoTIFF",
                   "processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"

diff --git a/requirements-cmr-token.txt b/requirements-cmr-token.txt
@@ -1,2 +1,2 @@
-boto3==1.26.118
-requests-pkcs12==1.15
+boto3==1.28.2
+requests-pkcs12==1.18
diff --git a/requirements-echo10-construction.txt b/requirements-echo10-construction.txt
@@ -1,3 +1,3 @@
-boto3==1.26.118
+boto3==1.28.2
 Jinja2==3.1.2
 shapely==2.0.1
diff --git a/requirements-echo10-to-cmr.txt b/requirements-echo10-to-cmr.txt
@@ -1,2 +1,2 @@
-boto3==1.26.118
-requests==2.28.2
+boto3==1.28.2
+requests==2.31.0
diff --git a/requirements-ingest.txt b/requirements-ingest.txt
@@ -1 +1 @@
-boto3==1.26.118
+boto3==1.28.2
diff --git a/requirements-invoke.txt b/requirements-invoke.txt
@@ -1 +1 @@
-boto3==1.26.118
+boto3==1.28.2
diff --git a/requirements-notify.txt b/requirements-notify.txt
@@ -1 +1 @@
-boto3==1.26.118
+boto3==1.28.2
diff --git a/requirements-verify.txt b/requirements-verify.txt
@@ -1,2 +1,2 @@
-boto3==1.26.118
-jsonschema==4.17.3
+boto3==1.28.2
+jsonschema==4.18.0
diff --git a/tests/data/v2/config.json b/tests/data/v2/config.json
@@ -46,7 +46,7 @@
 			]
 		}],
 		"collection": {
-			"dataset_id": "Sentinel-1 Interferograms (BETA)",
+			"dataset_id": "Sentinel-1 Interferograms",
 			"processing_type": "GUNW_STD",
 			"processing_type_display": "Standard Product, NetCDF",
 			"processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
@@ -55,28 +55,28 @@
 	"derived_products": [{
 		"label": "unwrappedPhase",
 		"layer": "science/grids/data/unwrappedPhase",
-		"dataset_id": "Sentinel-1 Interferograms - Unwrapped Phase (BETA)",
+		"dataset_id": "Sentinel-1 Interferograms - Unwrapped Phase",
 		"processing_type": "GUNW_UNW",
 		"processing_type_display": "Unwrapped Phase, GeoTIFF",
 		"processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
 	}, {
 		"label": "coherence",
 		"layer": "science/grids/data/coherence",
-		"dataset_id": "Sentinel-1 Interferograms - Coherence (BETA)",
+		"dataset_id": "Sentinel-1 Interferograms - Coherence",
 		"processing_type": "GUNW_COH",
 		"processing_type_display": "Coherence, GeoTIFF",
 		"processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
 	}, {
 		"label": "amplitude",
 		"layer": "science/grids/data/amplitude",
-		"dataset_id": "Sentinel-1 Interferograms - Amplitude (BETA)",
+		"dataset_id": "Sentinel-1 Interferograms - Amplitude",
 		"processing_type": "GUNW_AMP",
 		"processing_type_display": "Amplitude, GeoTIFF",
 		"processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
 	}, {
 		"label": "connectedComponents",
 		"layer": "science/grids/data/connectedComponents",
-		"dataset_id": "Sentinel-1 Interferograms - Connected Components (BETA)",
+		"dataset_id": "Sentinel-1 Interferograms - Connected Components",
 		"processing_type": "GUNW_CON",
 		"processing_type_display": "Connected Components, GeoTIFF",
 		"processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"

diff --git a/tests/data/v2/granule-amplitude.echo10 b/tests/data/v2/granule-amplitude.echo10
@@ -3,7 +3,7 @@
     <InsertTime>2023-04-07T18:23:39Z</InsertTime>
     <LastUpdate>2023-04-07T18:23:39Z</LastUpdate>
     <Collection>
-        <DataSetId>Sentinel-1 Interferograms - Amplitude (BETA)</DataSetId>
+        <DataSetId>Sentinel-1 Interferograms - Amplitude</DataSetId>
     </Collection>
     <DataGranule>
 

diff --git a/tests/data/v2/granule-coherence.echo10 b/tests/data/v2/granule-coherence.echo10
@@ -3,7 +3,7 @@
     <InsertTime>2023-04-07T18:23:39Z</InsertTime>
     <LastUpdate>2023-04-07T18:23:39Z</LastUpdate>
     <Collection>
-        <DataSetId>Sentinel-1 Interferograms - Coherence (BETA)</DataSetId>
+        <DataSetId>Sentinel-1 Interferograms - Coherence</DataSetId>
     </Collection>
     <DataGranule>
 

diff --git a/tests/data/v2/granule-connectedComponents.echo10 b/tests/data/v2/granule-connectedComponents.echo10
@@ -3,7 +3,7 @@
     <InsertTime>2023-04-07T18:23:39Z</InsertTime>
     <LastUpdate>2023-04-07T18:23:39Z</LastUpdate>
     <Collection>
-        <DataSetId>Sentinel-1 Interferograms - Connected Components (BETA)</DataSetId>
+        <DataSetId>Sentinel-1 Interferograms - Connected Components</DataSetId>
     </Collection>
     <DataGranule>
 

diff --git a/tests/data/v2/granule-unwrappedPhase.echo10 b/tests/data/v2/granule-unwrappedPhase.echo10
@@ -3,7 +3,7 @@
     <InsertTime>2023-04-07T18:23:39Z</InsertTime>
     <LastUpdate>2023-04-07T18:23:39Z</LastUpdate>
     <Collection>
-        <DataSetId>Sentinel-1 Interferograms - Unwrapped Phase (BETA)</DataSetId>
+        <DataSetId>Sentinel-1 Interferograms - Unwrapped Phase</DataSetId>
     </Collection>
     <DataGranule>
 

diff --git a/tests/data/v2/granule.echo10 b/tests/data/v2/granule.echo10
@@ -3,7 +3,7 @@
     <InsertTime>2023-04-07T18:23:39Z</InsertTime>
     <LastUpdate>2023-04-07T18:23:39Z</LastUpdate>
     <Collection>
-        <DataSetId>Sentinel-1 Interferograms (BETA)</DataSetId>
+        <DataSetId>Sentinel-1 Interferograms</DataSetId>
     </Collection>
     <DataGranule>
 

diff --git a/tests/data/v2/granule_data.json b/tests/data/v2/granule_data.json
@@ -2,7 +2,7 @@
   "granule_ur": "S1-GUNW-D-R-059-tops-20201118_20201013-180252-00179W_00051N-PP-1ec8-v2_0_6",
   "insert_time": "2023-04-07T18:23:39Z",
   "last_update": "2023-04-07T18:23:39Z",
-  "collection": "Sentinel-1 Interferograms (BETA)",
+  "collection": "Sentinel-1 Interferograms",
   "size_mb_data_granule": 46.92458248138428,
   "producer_granule_id": "S1-GUNW-D-R-059-tops-20201118_20201013-180252-00179W_00051N-PP-1ec8-v2_0_6",
   "production_date_time": "2023-03-23T00:34:17.970611Z",

diff --git a/tests/data/v3/config.json b/tests/data/v3/config.json
@@ -46,7 +46,7 @@
 			]
 		}],
 		"collection": {
-			"dataset_id": "Sentinel-1 Interferograms (BETA)",
+			"dataset_id": "Sentinel-1 Interferograms",
 			"processing_type": "GUNW_STD",
 			"processing_type_display": "Standard Product, NetCDF",
 			"processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
@@ -55,28 +55,28 @@
 	"derived_products": [{
 		"label": "unwrappedPhase",
 		"layer": "science/grids/data/unwrappedPhase",
-		"dataset_id": "Sentinel-1 Interferograms - Unwrapped Phase (BETA)",
+		"dataset_id": "Sentinel-1 Interferograms - Unwrapped Phase",
 		"processing_type": "GUNW_UNW",
 		"processing_type_display": "Unwrapped Phase, GeoTIFF",
 		"processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
 	}, {
 		"label": "coherence",
 		"layer": "science/grids/data/coherence",
-		"dataset_id": "Sentinel-1 Interferograms - Coherence (BETA)",
+		"dataset_id": "Sentinel-1 Interferograms - Coherence",
 		"processing_type": "GUNW_COH",
 		"processing_type_display": "Coherence, GeoTIFF",
 		"processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
 	}, {
 		"label": "amplitude",
 		"layer": "science/grids/data/amplitude",
-		"dataset_id": "Sentinel-1 Interferograms - Amplitude (BETA)",
+		"dataset_id": "Sentinel-1 Interferograms - Amplitude",
 		"processing_type": "GUNW_AMP",
 		"processing_type_display": "Amplitude, GeoTIFF",
 		"processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"
 	}, {
 		"label": "connectedComponents",
 		"layer": "science/grids/data/connectedComponents",
-		"dataset_id": "Sentinel-1 Interferograms - Connected Components (BETA)",
+		"dataset_id": "Sentinel-1 Interferograms - Connected Components",
 		"processing_type": "GUNW_CON",
 		"processing_type_display": "Connected Components, GeoTIFF",
 		"processing_description": "Sentinel-1 SLC interferometric products generated by JPL using ISCE v2.0.0, delivered by ASF"

diff --git a/tests/data/v3/granule-amplitude.echo10 b/tests/data/v3/granule-amplitude.echo10
@@ -3,7 +3,7 @@
     <InsertTime>2023-04-07T18:23:39Z</InsertTime>
     <LastUpdate>2023-04-07T18:23:39Z</LastUpdate>
     <Collection>
-        <DataSetId>Sentinel-1 Interferograms - Amplitude (BETA)</DataSetId>
+        <DataSetId>Sentinel-1 Interferograms - Amplitude</DataSetId>
     </Collection>
     <DataGranule>
 

diff --git a/tests/data/v3/granule-coherence.echo10 b/tests/data/v3/granule-coherence.echo10
@@ -3,7 +3,7 @@
     <InsertTime>2023-04-07T18:23:39Z</InsertTime>
     <LastUpdate>2023-04-07T18:23:39Z</LastUpdate>
     <Collection>
-        <DataSetId>Sentinel-1 Interferograms - Coherence (BETA)</DataSetId>
+        <DataSetId>Sentinel-1 Interferograms - Coherence</DataSetId>
     </Collection>
     <DataGranule>
 

diff --git a/tests/data/v3/granule-connectedComponents.echo10 b/tests/data/v3/granule-connectedComponents.echo10
@@ -3,7 +3,7 @@
     <InsertTime>2023-04-07T18:23:39Z</InsertTime>
     <LastUpdate>2023-04-07T18:23:39Z</LastUpdate>
     <Collection>
-        <DataSetId>Sentinel-1 Interferograms - Connected Components (BETA)</DataSetId>
+        <DataSetId>Sentinel-1 Interferograms - Connected Components</DataSetId>
     </Collection>
     <DataGranule>
 

diff --git a/tests/data/v3/granule-unwrappedPhase.echo10 b/tests/data/v3/granule-unwrappedPhase.echo10
@@ -3,7 +3,7 @@
     <InsertTime>2023-04-07T18:23:39Z</InsertTime>
     <LastUpdate>2023-04-07T18:23:39Z</LastUpdate>
     <Collection>
-        <DataSetId>Sentinel-1 Interferograms - Unwrapped Phase (BETA)</DataSetId>
+        <DataSetId>Sentinel-1 Interferograms - Unwrapped Phase</DataSetId>
     </Collection>
     <DataGranule>
 

diff --git a/tests/data/v3/granule.echo10 b/tests/data/v3/granule.echo10
@@ -3,7 +3,7 @@
     <InsertTime>2023-04-07T18:23:39Z</InsertTime>
     <LastUpdate>2023-04-07T18:23:39Z</LastUpdate>
     <Collection>
-        <DataSetId>Sentinel-1 Interferograms (BETA)</DataSetId>
+        <DataSetId>Sentinel-1 Interferograms</DataSetId>
     </Collection>
     <DataGranule>
 

diff --git a/tests/data/v3/granule_data.json b/tests/data/v3/granule_data.json
@@ -2,7 +2,7 @@
     "granule_ur": "S1-GUNW-A-R-072-tops-20171117_20171111-145939-00043E_00034N-PP-428e-v3_0_0",
     "insert_time": "2023-04-07T18:23:39Z",
     "last_update": "2023-04-07T18:23:39Z",
-    "collection": "Sentinel-1 Interferograms (BETA)",
+    "collection": "Sentinel-1 Interferograms",
     "size_mb_data_granule": 46.92458248138428,
     "producer_granule_id": "S1-GUNW-A-R-072-tops-20171117_20171111-145939-00043E_00034N-PP-428e-v3_0_0",
     "production_date_time": "2023-04-13T17:31:53.185420Z",

diff --git a/tests/example-message.json b/tests/example-message.json
@@ -0,0 +1,20 @@
+{
+  "ProductName": "S1-GUNW-A-R-014-tops-20230116_20220214-153036-00038E_00022N-PP-3708-v2_0_6",
+  "DeliveryTime": "2023-04-19T20:06:35.712653",
+  "ResponseTopic": {
+    "Region": "us-east-1",
+    "Arn": "arn:aws:sns:us-east-1:406893895021:hyp3-ingest-responses"
+  },
+  "Browse": {
+    "Bucket": "grfn-test-data",
+    "Key": "S1-GUNW-A-R-014-tops-20230116_20220214-153036-00038E_00022N-PP-3708-v2_0_6.png"
+  },
+  "Metadata": {
+    "Bucket": "grfn-test-data",
+    "Key": "S1-GUNW-A-R-014-tops-20230116_20220214-153036-00038E_00022N-PP-3708-v2_0_6.json"
+  },
+  "Product": {
+    "Bucket": "grfn-test-data",
+    "Key": "S1-GUNW-A-R-014-tops-20230116_20220214-153036-00038E_00022N-PP-3708-v2_0_6.nc"
+  }
+}