Merge pull request #85 from MikroElektronika/improvement/update-db-fr…

…om-mikrosdk Final update for database changes
MikroElektronika · Aug 23, 2024 · 94306bf · 94306bf
2 parents 4a55720 + a00f5bb
commit 94306bf
Show file tree

Hide file tree

Showing 5 changed files with 518 additions and 63 deletions.
diff --git a/.github/workflows/updateDb.yaml b/.github/workflows/updateDb.yaml
@@ -12,8 +12,17 @@ jobs:
   Update-Database:
     runs-on: ubuntu-latest
     steps:
+      - name: Authorize Mikroe Actions App
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.MIKROE_ACTIONS }}
+          private-key: ${{ secrets.MIKROE_ACTIONS_KEY }}
+
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          token: ${{ steps.app-token.outputs.token }}
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -31,6 +40,22 @@ jobs:
         run: |
           python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ github.event.inputs.release_version }} "latest"
 
+      - name: Add GitHub Actions credentials
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+
+      - name: Commit and Push Changes
+        run: |
+          if [ -n "$(git status --porcelain)" ]; then
+            echo "Updating with new CHANGELOG.md";
+            git add necto_db.db
+            git commit -m "Updated necto database with latest merged release."
+            git push
+          else
+            echo "No changes made to necto_db.db";
+          fi
+
       - name: Run Index Script
         env:
           ES_HOST: ${{ secrets.ES_HOST }}

diff --git a/.github/workflows/updateDbFromSdk.yaml b/.github/workflows/updateDbFromSdk.yaml
@@ -1,4 +1,4 @@
-name: Update Latest Database
+name: Update Latest Database From mikroSDK
 
 on:
   repository_dispatch:
@@ -8,10 +8,18 @@ jobs:
   Update-Database-From-mikroSDK:
     runs-on: ubuntu-latest
     steps:
+      - name: Authorize Mikroe Actions App
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.MIKROE_ACTIONS }}
+          private-key: ${{ secrets.MIKROE_ACTIONS_KEY }}
+
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.client_payload.sha }}
+          token: ${{ steps.app-token.outputs.token }}
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -29,19 +37,35 @@ jobs:
         id: mikrosdk_payload
         run: |
           tag_version=${{ github.event.client_payload.version }}
-          echo "Using mikroSDK version $version to update database."
-
-    #   - name: Upload new database asset
-    #     run: |
-    #       python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ github.event.inputs.release_version }} ${{ steps.mikrosdk_payload.outputs.tag_version }}
-
-    #   - name: Run Index Script
-    #     env:
-    #       ES_HOST: ${{ secrets.ES_HOST }}
-    #       ES_USER: ${{ secrets.ES_USER }}
-    #       ES_PASSWORD: ${{ secrets.ES_PASSWORD }}
-    #     run: |
-    #       echo "Indexing database to TEST."
-    #       python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} False ${{ github.event.inputs.release_version }} True
-    #       echo "Indexing database to LIVE."
-    #       python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_LIVE }} False ${{ github.event.inputs.release_version }} True
+          echo "Using mikroSDK version $tag_version to update database."
+
+      - name: Upload new database asset
+        run: |
+          python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP }} ${{ secrets.PROG_DEBUG_MIKROPROG }} "latest" ${{ steps.mikrosdk_payload.outputs.tag_version }}
+
+      - name: Add GitHub Actions credentials
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+
+      - name: Commit and Push Changes
+        run: |
+          if [ -n "$(git status --porcelain)" ]; then
+            echo "Updating with new CHANGELOG.md";
+            git add necto_db.db
+            git commit -m "Updated necto database with latest merged release."
+            git push
+          else
+            echo "No changes made to necto_db.db";
+          fi
+
+      - name: Run Index Script
+        env:
+          ES_HOST: ${{ secrets.ES_HOST }}
+          ES_USER: ${{ secrets.ES_USER }}
+          ES_PASSWORD: ${{ secrets.ES_PASSWORD }}
+        run: |
+          echo "Indexing database to TEST."
+          python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} False ${{ github.event.inputs.release_version }} True
+          echo "Indexing database to LIVE."
+          python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_LIVE }} False ${{ github.event.inputs.release_version }} True
diff --git a/.gitignore b/.gitignore
@@ -6,3 +6,4 @@ output
 __pycache__
 *.7z
 **/*/necto_db.db
+tmp
diff --git a/scripts/addSdkVersion.py b/scripts/addSdkVersion.py
@@ -0,0 +1,194 @@
+import os, re, sys, \
+       shutil, argparse, \
+       sqlite3
+
+from pathlib import Path
+
+## Import utility modules
+## Append to system path
+sys.path.append(str(Path(os.path.dirname(__file__)).parent.parent.absolute()))
+sys.path.append(str(Path(os.path.dirname(__file__)).absolute()))
+
+import enums as enums
+import support as utility
+
+def functionRegex(value, pattern):
+    reg = re.compile(value)
+    return reg.search(pattern) is not None
+
+def read_data_from_db(db, sql_query):
+    ## Open the database / connect to it
+    con = sqlite3.connect(db)
+    cur = con.cursor()
+
+    ## Create the REGEXP function to be used in DB
+    con.create_function("REGEXP", 2, functionRegex)
+
+    ## Execute the desired query
+    results = cur.execute(sql_query).fetchall()
+    # results = cur.fetchall()
+
+    ## Close the connection
+    cur.close()
+    con.close()
+
+    ## Return query results
+    return len(results), results
+
+def insertIntoTable(db, tableName, values, columns):
+    import sqlite3
+
+    conn = sqlite3.connect(db)
+    cur = conn.cursor()
+    numOfItems = ''
+    for itemCount in range(1, len(values) + 1):
+        numOfItems += '?,'
+    cur.execute(f'INSERT OR IGNORE INTO {tableName} ({columns}) VALUES ({numOfItems[:-1]})', values)
+    conn.commit()
+    conn.close()
+
+## Download databases or fetch from disk
+def downloadDb(downloadLink, overwrite=True):
+    dbPath = None
+    if 'http' in downloadLink:
+        if '.7z' in downloadLink:
+            dbPath = os.path.join(os.path.dirname(__file__), "dbGithub.db")
+            if overwrite or not os.path.isfile(dbPath):
+                utility.extract_archive_from_url(
+                    downloadLink, os.path.join(os.path.dirname(__file__), "dbGithub")
+                )
+                shutil.move(
+                    os.path.join(os.path.dirname(__file__), "dbGithub/necto_db.db"),
+                    os.path.join(os.path.dirname(__file__), "dbGithub.db")
+                )
+            if os.path.exists(os.path.join(os.path.dirname(__file__), "dbGithub")):
+                shutil.rmtree(os.path.join(os.path.dirname(__file__), "dbGithub"))
+    else:
+        dbPath = downloadLink ## Assume it is a local literal path
+
+    return dbPath
+
+def filter_versions(versions):
+    # Filter out versions that contain non-numeric characters (e.g., words or suffixes)
+    filtered_versions = [v for v in versions if all(part.isdigit() for part in v.split('.'))]
+    return filtered_versions
+
+def get_highest_and_second_highest(versions):
+    from packaging import version
+    # Parse the version strings to version objects for comparison
+    version_objects = [version.parse(v) for v in versions]
+
+    # Sort the versions in descending order
+    sorted_versions = sorted(version_objects, reverse=True)
+
+    # Get the highest and second-highest versions
+    highest_version = str(sorted_versions[0])
+    second_highest_version = str(sorted_versions[1]) if len(sorted_versions) > 1 else None
+
+    return highest_version, second_highest_version
+
+def addSdkVersion(database, sdkVersion):
+    sdkVersionUid = None
+    sdkVersionUidPrevious = None
+    isSdkVersionPresent = read_data_from_db(
+        database, f'SELECT DISTINCT uid, version FROM SDKs WHERE version = "{sdkVersion}"'
+    )
+    if not isSdkVersionPresent[enums.dbSync.COUNT.value]:
+        SDKsCollumns = 'uid, sdk_development_kit, name, legacy, icon, version, installed'
+        insertIntoTable(
+            database,
+            'SDKs',
+            [
+                f'mikrosdk_v{sdkVersion.replace('.','')}',
+                0,
+                'mikroSDK',
+                0,
+                'images/mikrosdk.png',
+                sdkVersion,
+                0
+            ],
+            SDKsCollumns
+        )
+        sdkVersionUid = f'mikrosdk_v{sdkVersion.replace('.','')}'
+
+        sdkVersions = read_data_from_db(
+            database, f'SELECT DISTINCT version FROM SDKs WHERE name IS "mikroSDK"'
+        )
+
+        versionList = []
+        for eachSdkVersion in sdkVersions[enums.dbSync.ELEMENTS.value]:
+            versionList.append(eachSdkVersion[0])
+        currentVersion, previousVersion = get_highest_and_second_highest(filter_versions(versionList))
+        sdkVersionUidPrevious = read_data_from_db(
+            database, f'SELECT uid FROM SDKs WHERE version = "{previousVersion}"'
+        )[enums.dbSync.ELEMENTS.value][0][0]
+
+    return sdkVersionUid, sdkVersionUidPrevious
+
+def insertIntoSdk(database, tableName, tableCollumn, sdkUidPrevious, sdkUidNew):
+    for eachTable, eachUid in zip(tableName, tableCollumn):
+        allFoundValues = read_data_from_db(
+            database, f'SELECT * FROM {eachTable} WHERE sdk_uid = "{sdkUidPrevious}"'
+        )
+        for eachValue in allFoundValues[enums.dbSync.ELEMENTS.value]:
+            formattedMessage = 'Inserted %s into %s table.\n' % (eachValue[enums.dbSync.ELEMENTS.value],eachTable)
+            print(formattedMessage)
+            insertIntoTable(
+                database,
+                eachTable,
+                [
+                    sdkUidNew,
+                    eachValue[enums.dbSync.ELEMENTS.value]
+                ],
+                f'sdk_uid, {eachUid}'
+            )
+
+## Main runner
+if __name__ == "__main__":
+    # First, check for arguments passed
+    parser = argparse.ArgumentParser(description='')
+    parser.add_argument(
+        '--sdkVersionByTag',
+        type=str,
+        default='',
+        help='GITHUB tag name used for SDK version.'
+    )
+
+    ## Parse the arguments
+    args = parser.parse_args()
+
+    ## Step 1 - if links passed, download the database first
+    database = downloadDb(
+        ## Always download database from latest release
+        'https://github.com/MikroElektronika/core_packages/releases/latest/download/database.7z',
+        False
+    )
+
+    ## Step 2 - add new sdk version
+    sdkVersionUidNew, sdkVersionUidPrevious = addSdkVersion(database, args.sdkVersionByTag)
+    ## Make sure to check if it exists already
+    if not sdkVersionUidNew:
+        raise ValueError('mikroSDK %s already exists in current database!' % args.sdkVersionByTag)
+
+    ## Step 3 - add data to tables
+    insertIntoSdk(
+        database,
+        [
+            'SDKToBoard',
+            'SDKToBuildSystem',
+            'SDKToCompiler',
+            'SDKToDevice',
+            'SDKToDisplay'
+        ],
+        [
+            'board_uid',
+            'build_system_uid',
+            'compiler_uid',
+            'device_uid',
+            'display_uid'
+        ],
+        sdkVersionUidPrevious,
+        sdkVersionUidNew
+    )
+    ## ------------------------------------------------------------------------------------ ##
+## EOF Main runner