moving sqlitePragmaString function to SqliteRepository file.

Migrated from github.com/glebarez/sqlite to gorm.io/driver/sqlite (which uses github.com/mattn/go-sqlite3)

Override github.com/mattn/go-sqlite3 with forked version from @jgiannuzzi which supports Encryption at rest.
See mattn/go-sqlite3#1109
See #236

Added documentation for how to open an encrypted sqlite database in IntelliJ - CONTRIBUTING.md update

CONTRIBUTING.md

@@ -235,3 +235,28 @@ curl -H "Authorization: Bearer ${JWT_TOKEN_HERE}" http://localhost:5984/_session
 ```bash
 ng run fastenhealth:storybook
 ```
+
+
+# Access Encrypted SQLite Database with IntelliJ
+
+- Download the latest `sqlite-jdbc-crypt` jar from https://github.com/Willena/sqlite-jdbc-crypt/releases
+- Open IntelliJ -> Data Source Properties -> Driver Tab
+- Find & Select `Sqlite` -> Right Click -> Duplicate
+- Rename to `Sqlite (Encrypted)`
+- Find `Driver Files` -> Select `sqlite-jdbc-crypt` jar that you downloaded previously
+- Remove `Xerial Sqlite JDBC` jar
+- Click `Apply` -> Click `OK`
+- Create New Data Source -> Select `Sqlite (Encrypted)` -> Change Connection Type to `Url only`
+- Specify the following connection url: `jdbc:sqlite:fasten.db?cipher=sqlcipher&legacy=3&hmac_use=0&kdf_iter=4000&legacy_page_size=1024&key=123456789012345678901234567890`
+- Replace `key` with the encryption key specified in your config file (`database.encryption_key`)
+- Click `Test Connection` -> Should be successful
+- Click `Apply` -> Click `OK`
+
+# Flush SQLite Write-Ahead-Log (WAL) to Database
+
+```sqlite
+PRAGMA wal_checkpoint(TRUNCATE);
+```
+
+See: https://sqlite.org/forum/info/fefd56014e2135589ea57825b0e2aa3e2df5daf53b5e41aa6a9d8f0c29d0b8e5
+TODO: check if https://www.sqlite.org/pragma.html#pragma_wal_checkpoint can be used to do this automatically. 

backend/pkg/database/gorm_common.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net/url"
 	"strings"
 	"time"
 
@@ -1119,19 +1118,6 @@ func (gr *GormRepository) CancelAllLockedBackgroundJobsAndFail() error {
 // Utilities
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-func sqlitePragmaString(pragmas map[string]string) string {
-	q := url.Values{}
-	for key, val := range pragmas {
-		q.Add("_pragma", fmt.Sprintf("%s=%s", key, val))
-	}
-
-	queryStr := q.Encode()
-	if len(queryStr) > 0 {
-		return "?" + queryStr
-	}
-	return ""
-}
-
 // Internal function
 // This function will return a list of resources from all FHIR tables in the database
 // The query allows us to set the  source id, source resource id, source resource type

backend/pkg/database/sqlite_repository.go

@@ -2,16 +2,19 @@ package database
 
 import (
 	"fmt"
+	"net/url"
 	"strings"
 
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/config"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/event_bus"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/models"
-	"github.com/glebarez/sqlite"
 	"github.com/sirupsen/logrus"
+	//"github.com/glebarez/sqlite"
+	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 )
 
+// uses github.com/mattn/go-sqlite3 driver (warning, uses CGO)
 func newSqliteRepository(appConfig config.Interface, globalLogger logrus.FieldLogger, eventBus event_bus.Interface) (DatabaseRepository, error) {
 	//backgroundContext := context.Background()
 
@@ -37,11 +40,28 @@ func newSqliteRepository(appConfig config.Interface, globalLogger logrus.FieldLo
 	// In this case all writes are appended to a temporary file (write-ahead log) and this file is periodically merged with the original database. When SQLite is searching for something it would first check this temporary file and if nothing is found proceed with the main database file.
 	// As a result, readers don’t compete with writers and performance is much better compared to the Old Way.
 	// https://stackoverflow.com/questions/4060772/sqlite-concurrent-access
-	pragmaStr := sqlitePragmaString(map[string]string{
-		"busy_timeout": "5000",
-		"foreign_keys": "ON",
-		"journal_mode": "wal",
-	})
+	//
+	// NOTE: this schema is driver specific, and may not work with other drivers.
+	// eg.https://github.com/mattn/go-sqlite3 uses `?_journal_mode=WAL` prefixes
+	// https://github.com/glebarez/sqlite uses `?_pragma=journal_mode(WAL)`
+	// see https://github.com/mattn/go-sqlite3/compare/master...jgiannuzzi:go-sqlite3:sqlite3mc
+	// see https://github.com/mattn/go-sqlite3/pull/1109
+	pragmaOpts := map[string]string{
+		"_busy_timeout": "5000",
+		"_foreign_keys": "on",
+		"_journal_mode": "WAL",
+	}
+	if appConfig.IsSet("database.encryption.key") {
+		pragmaOpts["_cipher"] = "sqlcipher"
+		pragmaOpts["_legacy"] = "3"
+		pragmaOpts["_hmac_use"] = "off"
+		pragmaOpts["_kdf_iter"] = "4000"
+		pragmaOpts["_legacy_page_size"] = "1024"
+		pragmaOpts["_key"] = appConfig.GetString("database.encryption.key")
+	}
+
+	pragmaStr := sqlitePragmaString(pragmaOpts)
+
 	dsn := "file:" + appConfig.GetString("database.location") + pragmaStr
 	database, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
 		//TODO: figure out how to log database queries again.
@@ -96,3 +116,17 @@ func newSqliteRepository(appConfig config.Interface, globalLogger logrus.FieldLo
 
 	return &fastenRepo, nil
 }
+
+func sqlitePragmaString(pragmas map[string]string) string {
+	q := url.Values{}
+	for key, val := range pragmas {
+		//q.Add("_pragma", fmt.Sprintf("%s=%s", key, val))
+		q.Add(key, val)
+	}
+
+	queryStr := q.Encode()
+	if len(queryStr) > 0 {
+		return "?" + queryStr
+	}
+	return ""
+}

go.mod

@@ -5,6 +5,13 @@ module github.com/fastenhealth/fasten-onprem
 
 go 1.18
 
+//replace github.com/fastenhealth/fasten-sources => ../fasten-sources
+//replace github.com/fastenhealth/gofhir-models => ../gofhir-models
+
+replace github.com/mattn/go-sqlite3 v1.14.17 => github.com/jgiannuzzi/go-sqlite3 v1.14.17-0.20230719111531-6e53453ccbd3
+
+//replace gorm.io/driver/sqlite v1.5.4 => github.com/jgiannuzzi/gorm-sqlite v1.4.4-0.20221215225833-42389ad31305
+
 require (
 	github.com/analogj/go-util v0.0.0-20210417161720-39b497cca03b
 	github.com/dave/jennifer v1.6.1
@@ -13,7 +20,6 @@ require (
 	github.com/fastenhealth/fasten-sources v0.4.9
 	github.com/fastenhealth/gofhir-models v0.0.6
 	github.com/gin-gonic/gin v1.9.0
-	github.com/glebarez/sqlite v1.5.0
 	github.com/go-gormigrate/gormigrate/v2 v2.1.1
 	github.com/golang-jwt/jwt/v4 v4.4.2
 	github.com/golang/mock v1.6.0
@@ -30,13 +36,15 @@ require (
 	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17
 	golang.org/x/net v0.14.0
 	gorm.io/datatypes v1.0.7
+	gorm.io/driver/sqlite v1.5.4
 	gorm.io/gorm v1.25.4
 )
 
 require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx/v5 v5.4.3 // indirect
+	github.com/mattn/go-sqlite3 v1.14.17 // indirect
 )
 
 require (
@@ -51,7 +59,6 @@ require (
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/glebarez/go-sqlite v1.19.1 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.13.0 // indirect
@@ -83,7 +90,6 @@ require (
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.7 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/seborama/govcr v4.5.0+incompatible // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
@@ -107,8 +113,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/driver/mysql v1.3.2 // indirect
 	gorm.io/driver/postgres v1.5.3
-	modernc.org/libc v1.19.0 // indirect
-	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.4.0 // indirect
-	modernc.org/sqlite v1.19.1 // indirect
 )