test migration

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

management/server/activity/cmd/main.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"context"
+
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/netbirdio/netbird/management/server/activity/sqlite"
+	log "github.com/sirupsen/logrus"
+)
+
+func main() {
+	encryptionKey := "<enc_key>"
+	eventsDBBase := "management/server/activity/cmd/events.db"
+
+	store, err := sqlite.NewSQLiteStore(context.Background(), eventsDBBase, encryptionKey)
+	if err != nil {
+		log.Fatalf("failed to create sqlite store: %v", err)
+	}
+
+	if err = store.GetLegacyEvents(); err != nil {
+		log.Fatalf("failed to get legacy events: %v", err)
+	}
+}

management/server/activity/sqlite/crypt.go

@@ -6,6 +6,7 @@ import (
 	"crypto/cipher"
 	"crypto/rand"
 	"encoding/base64"
+	"errors"
 	"fmt"
 )
 
@@ -13,6 +14,7 @@ var iv = []byte{10, 22, 13, 79, 05, 8, 52, 91, 87, 98, 88, 98, 35, 25, 13, 05}
 
 type FieldEncrypt struct {
 	block cipher.Block
+	gcm   cipher.AEAD
 }
 
 func GenerateKey() (string, error) {
@@ -35,14 +37,21 @@ func NewFieldEncrypt(key string) (*FieldEncrypt, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
 	ec := &FieldEncrypt{
 		block: block,
+		gcm:   gcm,
 	}
 
 	return ec, nil
 }
 
-func (ec *FieldEncrypt) Encrypt(payload string) string {
+func (ec *FieldEncrypt) LegacyEncrypt(payload string) string {
 	plainText := pkcs5Padding([]byte(payload))
 	cipherText := make([]byte, len(plainText))
 	cbc := cipher.NewCBCEncrypter(ec.block, iv)
@@ -50,7 +59,22 @@ func (ec *FieldEncrypt) Encrypt(payload string) string {
 	return base64.StdEncoding.EncodeToString(cipherText)
 }
 
-func (ec *FieldEncrypt) Decrypt(data string) (string, error) {
+// Encrypt encrypts plaintext using AES-GCM
+func (ec *FieldEncrypt) Encrypt(payload string) (string, error) {
+	plaintext := []byte(payload)
+	nonceSize := ec.gcm.NonceSize()
+
+	nonce := make([]byte, nonceSize, len(plaintext)+nonceSize+ec.gcm.Overhead())
+	if _, err := rand.Read(nonce); err != nil {
+		return "", err
+	}
+
+	ciphertext := ec.gcm.Seal(nonce, nonce, plaintext, nil)
+
+	return base64.StdEncoding.EncodeToString(ciphertext), nil
+}
+
+func (ec *FieldEncrypt) LegacyDecrypt(data string) (string, error) {
 	cipherText, err := base64.StdEncoding.DecodeString(data)
 	if err != nil {
 		return "", err
@@ -65,6 +89,27 @@ func (ec *FieldEncrypt) Decrypt(data string) (string, error) {
 	return string(payload), nil
 }
 
+// Decrypt decrypts ciphertext using AES-GCM
+func (ec *FieldEncrypt) Decrypt(data string) (string, error) {
+	cipherText, err := base64.StdEncoding.DecodeString(data)
+	if err != nil {
+		return "", err
+	}
+
+	nonceSize := ec.gcm.NonceSize()
+	if len(cipherText) < nonceSize {
+		return "", errors.New("cipher text too short")
+	}
+
+	nonce, cipherText := cipherText[:nonceSize], cipherText[nonceSize:]
+	plainText, err := ec.gcm.Open(nil, nonce, cipherText, nil)
+	if err != nil {
+		return "", err
+	}
+
+	return string(plainText), nil
+}
+
 func pkcs5Padding(ciphertext []byte) []byte {
 	padding := aes.BlockSize - len(ciphertext)%aes.BlockSize
 	padText := bytes.Repeat([]byte{byte(padding)}, padding)

management/server/activity/sqlite/crypt_test.go

@@ -15,7 +15,11 @@ func TestGenerateKey(t *testing.T) {
 		t.Fatalf("failed to init email encryption: %s", err)
 	}
 
-	encrypted := ee.Encrypt(testData)
+	encrypted, err := ee.Encrypt(testData)
+	if err != nil {
+		t.Fatalf("failed to encrypt data: %s", err)
+	}
+
 	if encrypted == "" {
 		t.Fatalf("invalid encrypted text")
 	}
@@ -30,6 +34,32 @@ func TestGenerateKey(t *testing.T) {
 	}
 }
 
+func TestGenerateKeyLegacy(t *testing.T) {
+	testData := "exampl@netbird.io"
+	key, err := GenerateKey()
+	if err != nil {
+		t.Fatalf("failed to generate key: %s", err)
+	}
+	ee, err := NewFieldEncrypt(key)
+	if err != nil {
+		t.Fatalf("failed to init email encryption: %s", err)
+	}
+
+	encrypted := ee.LegacyEncrypt(testData)
+	if encrypted == "" {
+		t.Fatalf("invalid encrypted text")
+	}
+
+	decrypted, err := ee.LegacyDecrypt(encrypted)
+	if err != nil {
+		t.Fatalf("failed to decrypt data: %s", err)
+	}
+
+	if decrypted != testData {
+		t.Fatalf("decrypted data is not match with test data: %s, %s", testData, decrypted)
+	}
+}
+
 func TestCorruptKey(t *testing.T) {
 	testData := "exampl@netbird.io"
 	key, err := GenerateKey()
@@ -41,7 +71,11 @@ func TestCorruptKey(t *testing.T) {
 		t.Fatalf("failed to init email encryption: %s", err)
 	}
 
-	encrypted := ee.Encrypt(testData)
+	encrypted, err := ee.Encrypt(testData)
+	if err != nil {
+		t.Fatalf("failed to encrypt data: %s", err)
+	}
+
 	if encrypted == "" {
 		t.Fatalf("invalid encrypted text")
 	}

management/server/activity/sqlite/legacy_events.go

@@ -0,0 +1,78 @@
+package sqlite
+
+import (
+	"database/sql"
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func (store *Store) GetLegacyEvents() error {
+	rows, err := store.db.Query(`SELECT id, email, name FROM deleted_users`)
+	if err != nil {
+		return fmt.Errorf("failed to execute select query: %v", err)
+	}
+	defer rows.Close()
+
+	if err = processLegacyEvents(store.fieldEncrypt, rows); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// processUserRows processes database rows of user data, decrypts legacy encryption fields, and re-encrypts them using GCM.
+func processLegacyEvents(crypt *FieldEncrypt, rows *sql.Rows) error {
+	var (
+		successCount int
+		failureCount int
+	)
+
+	for rows.Next() {
+		var (
+			id          string
+			email, name *string
+		)
+
+		err := rows.Scan(&id, &email, &name)
+		if err != nil {
+			return err
+		}
+
+		if email != nil {
+			_, err = crypt.LegacyDecrypt(*email)
+			if err != nil {
+				log.Warnf("failed to decrypt email for user %s: %v",
+					id,
+					fmt.Errorf("failed to decrypt email: %w", err),
+				)
+				failureCount++
+				continue
+			}
+		}
+
+		if name != nil {
+			_, err = crypt.LegacyDecrypt(*name)
+			if err != nil {
+				log.Warnf("failed to decrypt name for user %s: %v",
+					id,
+					fmt.Errorf("failed to decrypt name: %w", err),
+				)
+				failureCount++
+				continue
+			}
+		}
+
+		successCount++
+
+	}
+
+	if err := rows.Err(); err != nil {
+		return err
+	}
+
+	log.Infof("Successfully decoded entries: %d", successCount)
+	log.Infof("Failed decoded entries: %d", failureCount)
+
+	return nil
+}

management/server/activity/sqlite/migration.go

@@ -0,0 +1,157 @@
+package sqlite
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func migrate(ctx context.Context, crypt *FieldEncrypt, db *sql.DB) error {
+	if _, err := db.Exec(createTableQuery); err != nil {
+		return err
+	}
+
+	if _, err := db.Exec(creatTableDeletedUsersQuery); err != nil {
+		return err
+	}
+
+	if err := updateDeletedUsersTable(ctx, db); err != nil {
+		return fmt.Errorf("failed to update deleted_users table: %v", err)
+	}
+
+	return migrateLegacyEncryptedUsersToGCM(ctx, crypt, db)
+}
+
+// updateDeletedUsersTable checks and updates the deleted_users table schema to ensure required columns exist.
+func updateDeletedUsersTable(ctx context.Context, db *sql.DB) error {
+	exists, err := checkColumnExists(db, "deleted_users", "name")
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		log.WithContext(ctx).Debug("Adding name column to the deleted_users table")
+
+		_, err = db.Exec(`ALTER TABLE deleted_users ADD COLUMN name TEXT;`)
+		if err != nil {
+			return err
+		}
+
+		log.WithContext(ctx).Debug("Successfully added name column to the deleted_users table")
+	}
+
+	exists, err = checkColumnExists(db, "deleted_users", "enc_algo")
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		log.WithContext(ctx).Debug("Adding enc_algo column to the deleted_users table")
+
+		_, err = db.Exec(`ALTER TABLE deleted_users ADD COLUMN enc_algo TEXT;`)
+		if err != nil {
+			return err
+		}
+
+		log.WithContext(ctx).Debug("Successfully added enc_algo column to the deleted_users table")
+	}
+
+	return nil
+}
+
+// migrateLegacyEncryptedUsersToGCM migrates previously encrypted data using,
+// legacy CBC encryption with a static IV to the new GCM encryption method.
+func migrateLegacyEncryptedUsersToGCM(ctx context.Context, crypt *FieldEncrypt, db *sql.DB) error {
+	log.WithContext(ctx).Debug("Migrating CBC encrypted deleted users to GCM")
+
+	tx, err := db.Begin()
+	if err != nil {
+		return fmt.Errorf("failed to begin transaction: %v", err)
+	}
+	defer func() {
+		_ = tx.Rollback()
+	}()
+
+	rows, err := tx.Query(fmt.Sprintf(`SELECT id, email, name FROM deleted_users where enc_algo IS NULL OR enc_algo != '%s'`, gcmEncAlgo))
+	if err != nil {
+		return fmt.Errorf("failed to execute select query: %v", err)
+	}
+	defer rows.Close()
+
+	updateStmt, err := tx.Prepare(`UPDATE deleted_users SET email = ?, name = ?, enc_algo = ? WHERE id = ?`)
+	if err != nil {
+		return fmt.Errorf("failed to prepare update statement: %v", err)
+	}
+	defer updateStmt.Close()
+
+	if err = processUserRows(ctx, crypt, rows, updateStmt); err != nil {
+		return err
+	}
+
+	if err = tx.Commit(); err != nil {
+		return fmt.Errorf("failed to commit transaction: %v", err)
+	}
+
+	log.WithContext(ctx).Debug("Successfully migrated CBC encrypted deleted users to GCM")
+	return nil
+}
+
+// processUserRows processes database rows of user data, decrypts legacy encryption fields, and re-encrypts them using GCM.
+func processUserRows(ctx context.Context, crypt *FieldEncrypt, rows *sql.Rows, updateStmt *sql.Stmt) error {
+	for rows.Next() {
+		var (
+			id, decryptedEmail, decryptedName string
+			email, name                       *string
+		)
+
+		err := rows.Scan(&id, &email, &name)
+		if err != nil {
+			return err
+		}
+
+		if email != nil {
+			decryptedEmail, err = crypt.LegacyDecrypt(*email)
+			if err != nil {
+				log.WithContext(ctx).Warnf("skipping migrating deleted user %s: %v",
+					id,
+					fmt.Errorf("failed to decrypt email: %w", err),
+				)
+				continue
+			}
+		}
+
+		if name != nil {
+			decryptedName, err = crypt.LegacyDecrypt(*name)
+			if err != nil {
+				log.WithContext(ctx).Warnf("skipping migrating deleted user %s: %v",
+					id,
+					fmt.Errorf("failed to decrypt name: %w", err),
+				)
+				continue
+			}
+		}
+
+		encryptedEmail, err := crypt.Encrypt(decryptedEmail)
+		if err != nil {
+			return fmt.Errorf("failed to encrypt email: %w", err)
+		}
+
+		encryptedName, err := crypt.Encrypt(decryptedName)
+		if err != nil {
+			return fmt.Errorf("failed to encrypt name: %w", err)
+		}
+
+		_, err = updateStmt.Exec(encryptedEmail, encryptedName, gcmEncAlgo, id)
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		return err
+	}
+
+	return nil
+}

management/server/activity/sqlite/migration_test.go

@@ -0,0 +1,84 @@
+package sqlite
+
+import (
+	"context"
+	"database/sql"
+	"path/filepath"
+	"testing"
+	"time"
+
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/netbirdio/netbird/management/server/activity"
+
+	"github.com/stretchr/testify/require"
+)
+
+func setupDatabase(t *testing.T) *sql.DB {
+	t.Helper()
+
+	dbFile := filepath.Join(t.TempDir(), eventSinkDB)
+	db, err := sql.Open("sqlite3", dbFile)
+	require.NoError(t, err, "Failed to open database")
+
+	t.Cleanup(func() {
+		_ = db.Close()
+	})
+
+	_, err = db.Exec(createTableQuery)
+	require.NoError(t, err, "Failed to create events table")
+
+	_, err = db.Exec(`CREATE TABLE deleted_users (id TEXT NOT NULL, email TEXT NOT NULL, name TEXT);`)
+	require.NoError(t, err, "Failed to create deleted_users table")
+
+	return db
+}
+
+func TestMigrate(t *testing.T) {
+	db := setupDatabase(t)
+
+	key, err := GenerateKey()
+	require.NoError(t, err, "Failed to generate key")
+
+	crypt, err := NewFieldEncrypt(key)
+	require.NoError(t, err, "Failed to initialize FieldEncrypt")
+
+	legacyEmail := crypt.LegacyEncrypt("testaccount@test.com")
+	legacyName := crypt.LegacyEncrypt("Test Account")
+
+	_, err = db.Exec(`INSERT INTO events(activity, timestamp, initiator_id, target_id, account_id, meta) VALUES(?, ?, ?, ?, ?, ?)`,
+		activity.UserDeleted, time.Now(), "initiatorID", "targetID", "accountID", "")
+	require.NoError(t, err, "Failed to insert event")
+
+	_, err = db.Exec(`INSERT INTO deleted_users(id, email, name) VALUES(?, ?, ?)`, "targetID", legacyEmail, legacyName)
+	require.NoError(t, err, "Failed to insert legacy encrypted data")
+
+	colExists, err := checkColumnExists(db, "deleted_users", "enc_algo")
+	require.NoError(t, err, "Failed to check if enc_algo column exists")
+	require.False(t, colExists, "enc_algo column should not exist before migration")
+
+	err = migrate(context.Background(), crypt, db)
+	require.NoError(t, err, "Migration failed")
+
+	colExists, err = checkColumnExists(db, "deleted_users", "enc_algo")
+	require.NoError(t, err, "Failed to check if enc_algo column exists after migration")
+	require.True(t, colExists, "enc_algo column should exist after migration")
+
+	var encAlgo string
+	err = db.QueryRow(`SELECT enc_algo FROM deleted_users LIMIT 1`, "").Scan(&encAlgo)
+	require.NoError(t, err, "Failed to select updated data")
+	require.Equal(t, gcmEncAlgo, encAlgo, "enc_algo should be set to 'GCM' after migration")
+
+	store, err := createStore(crypt, db)
+	require.NoError(t, err, "Failed to create store")
+
+	events, err := store.Get(context.Background(), "accountID", 0, 1, false)
+	require.NoError(t, err, "Failed to get events")
+
+	require.Len(t, events, 1, "Should have one event")
+	require.Equal(t, activity.UserDeleted, events[0].Activity, "activity should match")
+	require.Equal(t, "initiatorID", events[0].InitiatorID, "initiator id should match")
+	require.Equal(t, "targetID", events[0].TargetID, "target id should match")
+	require.Equal(t, "accountID", events[0].AccountID, "account id should match")
+	require.Equal(t, "testaccount@test.com", events[0].Meta["email"], "email should match")
+	require.Equal(t, "Test Account", events[0].Meta["username"], "username should match")
+}

management/server/activity/sqlite/sqlite.go

@@ -5,7 +5,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
-	"path/filepath"
+	"os"
 	"time"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -73,6 +73,8 @@ const (
 
 	fallbackName  = "unknown"
 	fallbackEmail = "unknown@unknown.com"
+
+	gcmEncAlgo = "GCM"
 )
 
 // Store is the implementation of the activity.Store interface backed by SQLite
@@ -87,9 +89,15 @@ type Store struct {
 }
 
 // NewSQLiteStore creates a new Store with an event table if not exists.
-func NewSQLiteStore(ctx context.Context, dataDir string, encryptionKey string) (*Store, error) {
-	dbFile := filepath.Join(dataDir, eventSinkDB)
-	db, err := sql.Open("sqlite3", dbFile)
+func NewSQLiteStore(ctx context.Context, dbPath string, encryptionKey string) (*Store, error) {
+	//dbFile := filepath.Join(dataDir, eventSinkDB)
+	stats, err := os.Stat(dbPath)
+	if err != nil {
+		return nil, err
+	}
+	_ = stats
+
+	db, err := sql.Open("sqlite3", dbPath)
 	if err != nil {
 		return nil, err
 	}
@@ -100,58 +108,12 @@ func NewSQLiteStore(ctx context.Context, dataDir string, encryptionKey string) (
 		return nil, err
 	}
 
-	_, err = db.Exec(createTableQuery)
-	if err != nil {
-		_ = db.Close()
-		return nil, err
-	}
+	//if err = migrate(ctx, crypt, db); err != nil {
+	//	_ = db.Close()
+	//	return nil, fmt.Errorf("events database migration: %w", err)
+	//}
 
-	_, err = db.Exec(creatTableDeletedUsersQuery)
-	if err != nil {
-		_ = db.Close()
-		return nil, err
-	}
-
-	err = updateDeletedUsersTable(ctx, db)
-	if err != nil {
-		_ = db.Close()
-		return nil, err
-	}
-
-	insertStmt, err := db.Prepare(insertQuery)
-	if err != nil {
-		_ = db.Close()
-		return nil, err
-	}
-
-	selectDescStmt, err := db.Prepare(selectDescQuery)
-	if err != nil {
-		_ = db.Close()
-		return nil, err
-	}
-
-	selectAscStmt, err := db.Prepare(selectAscQuery)
-	if err != nil {
-		_ = db.Close()
-		return nil, err
-	}
-
-	deleteUserStmt, err := db.Prepare(insertDeleteUserQuery)
-	if err != nil {
-		_ = db.Close()
-		return nil, err
-	}
-
-	s := &Store{
-		db:                  db,
-		fieldEncrypt:        crypt,
-		insertStatement:     insertStmt,
-		selectDescStatement: selectDescStmt,
-		selectAscStatement:  selectAscStmt,
-		deleteUserStmt:      deleteUserStmt,
-	}
-
-	return s, nil
+	return createStore(crypt, db)
 }
 
 func (store *Store) processResult(ctx context.Context, result *sql.Rows) ([]*activity.Event, error) {
@@ -302,9 +264,16 @@ func (store *Store) saveDeletedUserEmailAndNameInEncrypted(event *activity.Event
 		return event.Meta, nil
 	}
 
-	encryptedEmail := store.fieldEncrypt.Encrypt(fmt.Sprintf("%s", email))
-	encryptedName := store.fieldEncrypt.Encrypt(fmt.Sprintf("%s", name))
-	_, err := store.deleteUserStmt.Exec(event.TargetID, encryptedEmail, encryptedName)
+	encryptedEmail, err := store.fieldEncrypt.Encrypt(fmt.Sprintf("%s", email))
+	if err != nil {
+		return nil, err
+	}
+	encryptedName, err := store.fieldEncrypt.Encrypt(fmt.Sprintf("%s", name))
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = store.deleteUserStmt.Exec(event.TargetID, encryptedEmail, encryptedName, gcmEncAlgo)
 	if err != nil {
 		return nil, err
 	}
@@ -325,43 +294,70 @@ func (store *Store) Close(_ context.Context) error {
 	return nil
 }
 
-func updateDeletedUsersTable(ctx context.Context, db *sql.DB) error {
-	log.WithContext(ctx).Debugf("check deleted_users table version")
-	rows, err := db.Query(`PRAGMA table_info(deleted_users);`)
+// createStore initializes and returns a new Store instance with prepared SQL statements.
+func createStore(crypt *FieldEncrypt, db *sql.DB) (*Store, error) {
+	insertStmt, err := db.Prepare(insertQuery)
 	if err != nil {
-		return err
+		_ = db.Close()
+		return nil, err
+	}
+
+	selectDescStmt, err := db.Prepare(selectDescQuery)
+	if err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+
+	selectAscStmt, err := db.Prepare(selectAscQuery)
+	if err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+
+	deleteUserStmt, err := db.Prepare(insertDeleteUserQuery)
+	if err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+
+	return &Store{
+		db:                  db,
+		fieldEncrypt:        crypt,
+		insertStatement:     insertStmt,
+		selectDescStatement: selectDescStmt,
+		selectAscStatement:  selectAscStmt,
+		deleteUserStmt:      deleteUserStmt,
+	}, nil
+}
+
+// checkColumnExists checks if a column exists in a specified table
+func checkColumnExists(db *sql.DB, tableName, columnName string) (bool, error) {
+	query := fmt.Sprintf("PRAGMA table_info(%s);", tableName)
+	rows, err := db.Query(query)
+	if err != nil {
+		return false, fmt.Errorf("failed to query table info: %w", err)
 	}
 	defer rows.Close()
-	found := false
+
 	for rows.Next() {
-		var (
-			cid      int
-			name     string
-			dataType string
-			notNull  int
-			dfltVal  sql.NullString
-			pk       int
-		)
-		err := rows.Scan(&cid, &name, &dataType, &notNull, &dfltVal, &pk)
+		var cid int
+		var name, ctype string
+		var notnull, pk int
+		var dfltValue sql.NullString
+
+		err = rows.Scan(&cid, &name, &ctype, &notnull, &dfltValue, &pk)
 		if err != nil {
-			return err
+			return false, fmt.Errorf("failed to scan row: %w", err)
 		}
-		if name == "name" {
-			found = true
-			break
+
+		if name == columnName {
+			return true, nil
 		}
 	}
 
-	err = rows.Err()
-	if err != nil {
-		return err
+	if err = rows.Err(); err != nil {
+		return false, err
 	}
 
-	if found {
-		return nil
-	}
-
-	log.WithContext(ctx).Debugf("update delted_users table")
-	_, err = db.Exec(`ALTER TABLE deleted_users ADD COLUMN name TEXT;`)
-	return err
+	return false, nil
 }