SEMICeu · bertvannuffelen · Jan 4, 2024 · Jan 4, 2024 · Jan 4, 2024 · Jan 4, 2024
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,80 +1,10 @@
 version: 2.1
 
 commands:
-  checkout-repositories:
-    description: "checkout the repositories and set correct publishing url"
-    parameters:
-      production_branch:
-        type: string
-        default: master
-      test_branch:
-        type: string
-        default: test
-      other_hostname:
-        type: string
-      test_hostname:
-        type: string
-      production_hostname:
-        type: string
-    steps:
-      - when:
-          condition:
-            equal: [ << pipeline.git.branch >>, <<parameters.production_branch>> ]
-          steps:
-            - run:
-                command: $PWD/scripts/checkoutRepositories.sh /tmp/workspace $PWD/config/publication.json $PWD/config
-                environment:
-                  HOSTNAME: <<parameters.production_hostname>>
-      - when:
-          condition:
-            equal: [ << pipeline.git.branch >>,<<parameters.test_branch>> ]
-          steps:
-            - run:
-                command: $PWD/scripts/checkoutRepositories.sh /tmp/workspace $PWD/config/publication.json $PWD/config
-                environment:
-                  HOSTNAME: <<parameters.test_hostname>>
-      - unless:
-          condition:
-            or:
-              - equal: [ << pipeline.git.branch >>,<<parameters.test_branch>> ]
-              - equal: [ << pipeline.git.branch >>,<<parameters.production_branch>> ]
-          steps:
-            - run:
-                command: $PWD/scripts/checkoutRepositories.sh /tmp/workspace $PWD/config/publication.json $PWD/config
-                environment:
-                  HOSTNAME: <<parameters.other_hostname>>
 jobs:
   checkout:
-    parameters:
-      production_branch:
-        type: string
-        default: master
-      test_branch:
-        type: string
-        default: test
-      production_folder:
-        type: string
-        default: production
-      test_folder:
-        type: string
-        default: test
-      other_folder:
-        type: string
-        default: dev
-      other_hostname:
-        type: string
-      test_hostname:
-        type: string
-      production_hostname:
-        type: string
     docker:
-      - image: circleci/python
-    environment:
-      PRODUCTION_BRANCH: <<parameters.production_branch>>
-      TEST_BRANCH: <<parameters.test_branch>>
-      PRODUCTION_FOLDER: <<parameters.production_folder>>
-      TEST_FOLDER: <<parameters.test_folder>>
-      OTHER_FOLDER: <<parameters.other_folder>>
+      - image: cimg/node:18.16.0
     steps:
       - checkout
       - add_ssh_keys:
@@ -83,19 +13,18 @@ jobs:
             - "0f:f5:f3:70:0a:47:30:b5:4d:c7:71:97:f9:63:9f:c9"
             - "f1:b9:c3:88:b2:23:a4:ab:9d:b0:b6:56:01:cb:e5:c8"
             - "ec:8f:5a:dc:dd:16:4f:7e:4a:ea:e5:9e:ac:f2:a1:a7"
-      - run: $PWD/scripts/update_sshconfig.sh private1
+      - run:
+          name: update and add extra statements of updating the ssh config for each private ssh key
+          command: |
+            $PWD/scripts/update_sshconfig.sh private1
       - run: $PWD/scripts/update_sshconfig.sh uri.semic.eu-thema
       - run: $PWD/scripts/update_sshconfig.sh Semicthema-DCAT-AP
       - run: mkdir -p workspace
       - run: ls -al workspace
       - run:
-          command: $PWD/scripts/findPublicationsToUpdate.sh /tmp/workspace $PWD/config/publication.json $PWD/  $PWD/config
-      - checkout-repositories:
-          production_branch: <<parameters.production_branch>>
-          test_branch: <<parameters.test_branch>>
-          other_hostname: <<parameters.other_hostname>>
-          test_hostname: <<parameters.test_hostname>>
-          production_hostname: <<parameters.production_hostname>>
+          command: $PWD/scripts/findPublicationsToUpdate.sh /tmp/workspace $PWD/config/publication.json $PWD/
+      - run: 
+          command: $PWD/scripts/checkoutRepositories.sh /tmp/workspace $PWD/config/publication.json $PWD/config
       - run: if [ -e /tmp/workspace/checkouts.txt ] ; then cat /tmp/workspace/checkouts.txt ; fi
       - run: if [ -e /tmp/workspace/rawcheckouts.txt ] ; then cat /tmp/workspace/rawcheckouts.txt ; fi
       - run: if [ -e /home/circleci/project/changes.txt ] ; then cat /home/circleci/project/changes.txt ; fi
@@ -157,7 +86,7 @@ jobs:
             [ $( egrep " ERROR " /tmp/workspace/report/*.report | wc -l ) == 0 ] || exit 1
   normalise-jsonld:
     docker:
-      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6
+      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9
     steps:
       - checkout
       - attach_workspace:
@@ -175,7 +104,7 @@ jobs:
             - src/
   render-translation-json:
     docker:
-      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6
+      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9
     steps:
       - checkout
       - attach_workspace:
@@ -201,7 +130,7 @@ jobs:
             - src/
   validate-and-generate-translation-report:
     docker:
-      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6
+      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9
     steps:
       - checkout
       - attach_workspace:
@@ -217,7 +146,7 @@ jobs:
             [ $( egrep " WARNING " /tmp/workspace/report/translation/*.report | wc -l ) == 0 ] || exit 1
   render-merged-jsonld:
     docker:
-      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6
+      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9
     steps:
       - checkout
       - attach_workspace:
@@ -261,7 +190,7 @@ jobs:
             - reporthtml/
   render-example-templates:
     docker:
-      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6
+      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9
     steps:
       - checkout
       - attach_workspace:
@@ -301,7 +230,7 @@ jobs:
             - target/
   render-xsd-details:
     docker:
-      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6
+      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9
     steps:
       - checkout
       - attach_workspace:
@@ -346,8 +275,6 @@ jobs:
             - report/
   convert-json-to-ttl:
     docker:
-            #- image: circleci/ruby:2.6.5-stretch
-            #- image: circleci/ruby:3.0.2-buster
       - image: cimg/ruby:3.0.2
     steps:
       - checkout
@@ -360,6 +287,7 @@ jobs:
             mkdir -p /tmp/workspace/target
       - run:
           name: Convert -SHACL.jsonld file to -SHACL.ttl files
+          no_output_timeout: 30m
           command: |
             find /tmp/workspace/target -name \*-SHACL.jsonld -type f | $PWD/scripts/convert_to_any.sh turtle
       - run:
@@ -373,7 +301,7 @@ jobs:
             - target/
   expand-links:
     docker:
-      - image: circleci/node:4.8.2
+      - image: cimg/node:18.16.0
     steps:
       - checkout
       - attach_workspace:
@@ -393,7 +321,7 @@ jobs:
             - target/
   render-voc-jsonld:
     docker:
-      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.7
+      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9
     steps:
       - checkout
       - attach_workspace:
@@ -433,18 +361,22 @@ jobs:
             touch /tmp/workspace/target/.ignore12
       - run:
           name: Convert vocabulary to turtle
+          no_output_timeout: 30m
           command: |
             find /tmp/workspace/target/doc/vocabularium -name \*.jsonld -type f | $PWD/scripts/convert_to_any.sh turtle
       - run:
           name: Convert vocabulary to ntriples
+          no_output_timeout: 30m
           command: |
             find /tmp/workspace/target/doc/vocabularium -name \*.jsonld -type f | $PWD/scripts/convert_to_any.sh ntriples
       - run:
           name: Convert vocabulary to rdf/xml
+          no_output_timeout: 30m
           command: |
             find /tmp/workspace/target/doc/vocabularium -name \*.jsonld -type f | $PWD/scripts/convert_to_any.sh rdfxml
       - run:
           name: list results
+          no_output_timeout: 30m
           command: |
             if [ -d /tmp/workspace/doc/vocabularium ]; then ls /tmp/workspace/doc/vocabularium; fi
       - persist_to_workspace:
@@ -453,7 +385,7 @@ jobs:
             - target/
   copy-raw:
     docker:
-      - image: circleci/python
+      - image: cimg/node:18.16.0
     steps:
       - checkout
       - attach_workspace:
@@ -475,23 +407,29 @@ jobs:
             - raw/
   create-artifact:
     parameters:
-      repository:
+      generatedReporFingerprint:
         type: string
     docker:
-      - image: circleci/node
+      - image: cimg/node:18.16.0
     steps:
       - checkout
       - attach_workspace:
           at: /tmp/workspace
       - add_ssh_keys:
           fingerprints:
+            - << parameters.generatedReporFingerprint >>
             - "6d:07:6e:01:36:b0:89:1d:b2:93:6e:bf:86:a1:4d:61"
+      - run:
+          name: update and add extra statements of updating the ssh config for each private ssh key
+          command: |
+            export GENERATEDREPO=$(jq -r .generatedrepository.repository $PWD/config/config.json)
+            $PWD/scripts/update_sshconfig.sh ${GENERATEDREPO}
       - run:
           name: Checkout target repo
-          workdir: /tmp
           command: |
-            rm -rf generated
-            git clone <<parameters.repository>> generated
+            rm -rf /tmp/generated
+            export REPO=$(jq -r .generatedrepository $PWD/config/config.json)
+            $PWD/scripts/checkoutRepoGithub.sh "${REPO}" /tmp/generated ${TOOLCHAIN_TOKEN}
       - run:
           name: clean the generated directory
           workdir: /tmp/generated
@@ -503,6 +441,13 @@ jobs:
           name: Copy static assets
           command: |
             cp -R site-skeleton/* /tmp/generated/
+#      - run:
+#          name: deploy statistics 
+#          command: |
+#           export CIRCLEWKD=$( eval echo "${CIRCLE_WORKING_DIRECTORY}" )
+#           pushd ${CIRCLEWKD}/statistics
+#           ./deploy.sh /tmp/generated
+#           popd
       - run:
           name: Copy generated assets
           workdir: /tmp/workspace
@@ -515,7 +460,7 @@ jobs:
             if [ -d translation ] ; then if [ ! "$(ls -A translation)" ] ; then echo "directory translation is empty" ; else cp -rT translation /tmp/generated/report/ ; fi fi
             if [ -d examples ] ; then if [ ! "$(ls -A examples)" ] ; then echo "directory examples is empty" ; else cp -rT examples /tmp/generated/examples/ ; fi fi
       - run:
-          name: List the files which have been created
+          name: check if all publication points are generated 
           workdir: /tmp/generated
           command: |
             ls -al *
@@ -540,8 +485,12 @@ jobs:
           name: Push results to github
           workdir: /tmp/generated
           command: |
-            git config user.email "[email protected]"
-            git config user.name "Circle CI Builder"
+            # expand the tilde first
+            export CIRCLEWKD=$( eval echo "${CIRCLE_WORKING_DIRECTORY}" )
+            export EMAIL=$(jq -r .generatedrepository.gitEmail $CIRCLEWKD/config/config.json)
+            export USER=$(jq -r .generatedrepository.gitUser $CIRCLEWKD/config/config.json)
+            git config user.email "${EMAIL}"
+            git config user.name "${USER}"
             git add .
             git status
             git commit -m "Applying changes from commit ${CIRCLE_SHA1}" --allow-empty
@@ -553,15 +502,7 @@ workflows:
   version: 2
   generate_documentation:
     jobs:
-      - checkout:
-          test_branch: test
-          production_branch: production
-          other_folder: dev
-          test_folder: test
-          production_folder: production
-          other_hostname: https://semiceu.github.io/
-          test_hostname: https://semiceu.github.io/
-          production_hostname: https://semiceu.github.io/
+      - checkout
       - extract-jsonld-details:
           requires:
             - checkout
@@ -617,6 +558,6 @@ workflows:
             - render-xsd-details
             - render-voc-rdf
       - create-artifact:
-          repository: [email protected]:SEMICeu/uri.semic.eu-generated.git
+          generatedReporFingerprint: "6d:07:6e:01:36:b0:89:1d:b2:93:6e:bf:86:a1:4d:61"
           requires:
             - expand-links
diff --git a/config/config.json b/config/config.json
@@ -3,18 +3,27 @@
   "otherLanguages": [
     "fr", "de"
   ],
-  "hostname": "https://semiceu.github.io/",
+  "hostname": "https://semiceu.github.io",
   "domain": "data.europa.eu",
   "publicationpoints" : ["dev"],
   "generatedrepository" : {
 	  "organisation": "SEMICeu",
 	  "repository" : "uri.semic.eu-generated",
-	  "private" : true
+	  "private" : true,
+	  "gitEmail" : "[email protected]",
+	  "gitUser" : "Circle CI Builder"
   },
-  "toolchainversion" : "3.0.5",
+  "toolchainversion" : "3.1.1",
   "toolchain" : {
 	  "strickness" : "lazy",
 	  "version": 3,
-	  "triggerall" : false
+	  "triggerall" : true
+  },
+  "shaclgenerator" : {
+	  "parameters": "-m individual -c 'uniqueLanguages' -c 'nodeKind'",
+	  "parameters-old": ""
+  },
+  "jsonldgenerator" : {
+	  "parameters" : "-d -l label"
   }
 }
diff --git a/config/context b/config/context
@@ -90,7 +90,7 @@
       "@type" : "@id"
     },
     "usage": {
-      "@id": "vann:usageNote",
+      "@id": "skos:scopeNote",
       "@container": "@language"
     },
     "parents" :{

diff --git a/config/dev/publication.json b/config/dev/publication.json
@@ -161,16 +161,16 @@
     "navigation": {
 		"prev": "https://semiceu.github.io/Core-Location-Vocabulary/releases/2.0.2"
 	},
-    "dummy": "75"
+    "dummy": "80"
   },
   {
     "urlref": "/doc/vocabularium/core-location",
     "repository": "[email protected]:SEMICeu/uri.semic.eu-thema.git",
-    "branchtag": "main",
+    "branchtag": "ReSpecCV",
     "name": "core-location",
     "filename": "config/core-location-voc.json",
     "navigation": {},
-    "dummy": "23"
+    "dummy": "24"
   },
   {
     "dummy": "8",

diff --git a/scripts/README.md b/scripts/README.md
@@ -6,8 +6,8 @@ The scripts are so designed that all configuration is present in the config dire
 
 Each script has as implicit requirement that the execution environment is the one in which is called as defined in CIRCLECI setup.
 Executing the scripts thus cannot be done directly after checking out this repository. 
-One has to create an execution environment (a Docker container) statisfying the needs.
-This can be done by mannualy executing the CIRCLECI configuration.
+One has to create an execution environment (a Docker container) satisfying the needs.
+This can be done by manually executing the CIRCLECI configuration.