Commit e603b936 authored by Ivar Derksen's avatar Ivar Derksen Committed by Sietse Ringers

Feat: client storage fully in bbolt

parent b6ab049f
......@@ -3,8 +3,8 @@ package irmaclient
import (
"encoding/binary"
"encoding/json"
"github.com/go-errors/errors"
"io/ioutil"
"os"
"path/filepath"
"time"
......@@ -39,7 +39,14 @@ const (
// Bucketnames bbolt
const (
logsBucket = "logs"
userdataBucket = "userdata" // Key/value: specified below
skKey = "sk" // Value: *secretKey
preferencesKey = "preferences" // Value: Preferences
updatesKey = "updates" // Value: []update
kssKey = "kss" // Value: map[irma.SchemeManagerIdentifier]*keyshareServer
attributesBucket = "attrs" // Key: irma.CredentialIdentifier, value: []*irma.AttributeList
logsBucket = "logs" // Key: (auto-increment index), value: *LogEntry
signaturesBucket = "sigs" // Key: credential.attrs.Hash, value: *gabi.CLSignature
)
func (s *storage) path(p string) string {
......@@ -63,7 +70,7 @@ func (s *storage) EnsureStorageExists() error {
return err
}
func (s *storage) load(dest interface{}, path string) (err error) {
func (s *storage) loadFromFile(dest interface{}, path string) (err error) {
exists, err := fs.PathExists(s.path(path))
if err != nil || !exists {
return
......@@ -83,6 +90,47 @@ func (s *storage) store(contents interface{}, file string) error {
return fs.SaveFile(s.path(file), bts)
}
func (s *storage) txStore(tx *bbolt.Tx, key interface{}, value interface{}, bucketName string) error {
b, err := tx.CreateBucketIfNotExists([]byte(bucketName))
if err != nil {
return err
}
btsKey, err := json.Marshal(key)
if err != nil {
return err
}
btsValue, err := json.Marshal(value)
if err != nil {
return err
}
return b.Put(btsKey, btsValue)
}
func (s *storage) txLoad(tx *bbolt.Tx, key interface{}, dest interface{}, bucketName string) (found bool, err error) {
b := tx.Bucket([]byte(bucketName))
if b == nil {
return false, nil
}
btsKey, err := json.Marshal(key)
if err != nil {
return false, err
}
bts := b.Get(btsKey)
if bts == nil {
return false, nil
}
return true, json.Unmarshal(bts, dest)
}
func (s *storage) load(key interface{}, dest interface{}, bucketName string) (found bool, err error) {
err = s.db.View(func(tx *bbolt.Tx) error {
found, err = s.txLoad(tx, key, dest, bucketName)
return err
})
return
}
func (s *storage) signatureFilename(attrs *irma.AttributeList) string {
// We take the SHA256 hash over all attributes as the filename for the signature.
// This means that the signatures of two credentials that have identical attributes
......@@ -93,34 +141,63 @@ func (s *storage) signatureFilename(attrs *irma.AttributeList) string {
}
func (s *storage) DeleteSignature(attrs *irma.AttributeList) error {
return os.Remove(s.path(s.signatureFilename(attrs)))
return s.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(signaturesBucket))
if err != nil {
return err
}
return b.Delete([]byte(attrs.Hash()))
})
}
func (s *storage) StoreSignature(cred *credential) error {
return s.store(cred.Signature, s.signatureFilename(cred.AttributeList()))
return s.db.Update(func(tx *bbolt.Tx) error {
return s.TxStoreSignature(tx, cred.attrs.Hash(), cred.Signature)
})
}
func (s *storage) TxStoreSignature(tx *bbolt.Tx, credHash string, sig *gabi.CLSignature) error {
// We take the SHA256 hash over all attributes as the bucket key for the signature.
// This means that of the signatures of two credentials that have identical attributes
// only one gets stored, one overwriting the other - but that doesn't
// matter, because either one of the signatures is valid over both attribute lists,
return s.txStore(tx, credHash, sig, signaturesBucket)
}
func (s *storage) StoreSecretKey(sk *secretKey) error {
return s.store(sk, skFile)
return s.db.Update(func(tx *bbolt.Tx) error {
return s.TxStoreSecretKey(tx, sk)
})
}
func (s *storage) TxStoreSecretKey(tx *bbolt.Tx, sk *secretKey) error {
return s.txStore(tx, skKey, sk, userdataBucket)
}
func (s *storage) StoreAttributes(attributes map[irma.CredentialTypeIdentifier][]*irma.AttributeList) error {
temp := []*irma.AttributeList{}
for _, attrlistlist := range attributes {
for _, attrlist := range attrlistlist {
temp = append(temp, attrlist)
return s.db.Update(func(tx *bbolt.Tx) error {
return s.TxStoreAttributes(tx, attributes)
})
}
func (s *storage) TxStoreAttributes(tx *bbolt.Tx, attrs map[irma.CredentialTypeIdentifier][]*irma.AttributeList) error {
for credTypeID, attrlistlist := range attrs {
err := s.txStore(tx, credTypeID.String(), attrlistlist, attributesBucket)
if err != nil {
return err
}
}
return s.store(temp, attributesFile)
return nil
}
func (s *storage) StoreKeyshareServers(keyshareServers map[irma.SchemeManagerIdentifier]*keyshareServer) error {
return s.store(keyshareServers, kssFile)
return s.db.Update(func(tx *bbolt.Tx) error {
return s.TxStoreKeyshareServers(tx, keyshareServers)
})
}
func (s *storage) StoreLogs(logs []*LogEntry) error {
return s.store(logs, logsFile)
func (s *storage) TxStoreKeyshareServers(tx *bbolt.Tx, keyshareServers map[irma.SchemeManagerIdentifier]*keyshareServer) error {
return s.txStore(tx, kssKey, &keyshareServers, userdataBucket)
}
func (s *storage) AddLogEntry(entry *LogEntry) error {
......@@ -152,36 +229,73 @@ func (s *storage) logEntryKeyToBytes(id uint64) []byte {
}
func (s *storage) StorePreferences(prefs Preferences) error {
return s.store(prefs, preferencesFile)
return s.db.Update(func(tx *bbolt.Tx) error {
return s.TxStorePreferences(tx, prefs)
})
}
func (s *storage) TxStorePreferences(tx *bbolt.Tx, prefs Preferences) error {
return s.txStore(tx, preferencesKey, prefs, userdataBucket)
}
func (s *storage) StoreUpdates(updates []update) (err error) {
return s.store(updates, updatesFile)
return s.db.Update(func(tx *bbolt.Tx) error {
return s.TxStoreUpdates(tx, updates)
})
}
func (s *storage) LoadSignature(attrs *irma.AttributeList) (signature *gabi.CLSignature, err error) {
func (s *storage) TxStoreUpdates(tx *bbolt.Tx, updates []update) error {
return s.txStore(tx, updatesKey, updates, userdataBucket)
}
func (s *storage) LoadSignatureFile(attrs *irma.AttributeList) (signature *gabi.CLSignature, err error) {
sigpath := s.signatureFilename(attrs)
if err := fs.AssertPathExists(s.path(sigpath)); err != nil {
return nil, err
}
signature = new(gabi.CLSignature)
if err := s.load(signature, sigpath); err != nil {
if err := s.loadFromFile(signature, sigpath); err != nil {
return nil, err
}
return signature, nil
}
// LoadSecretKey retrieves and returns the secret key from storage, or if no secret key
// was found in storage, it generates, saves, and returns a new secret key.
func (s *storage) LoadSecretKey() (*secretKey, error) {
func (s *storage) LoadSignature(attrs *irma.AttributeList) (signature *gabi.CLSignature, err error) {
signature = new(gabi.CLSignature)
found, err := s.load(attrs.Hash(), signature, signaturesBucket)
if err != nil {
return nil, err
} else if !found {
return nil, errors.Errorf("Signature of credential with hash %s cannot be found", attrs.Hash())
}
return
}
// LoadSecretKeyFile retrieves and returns the secret key from file storage. When no secret key
// file is found, nil is returned.
func (s *storage) LoadSecretKeyFile() (*secretKey, error) {
var err error
sk := &secretKey{}
if err = s.load(sk, skFile); err != nil {
if err = s.loadFromFile(sk, skFile); err != nil {
return nil, err
}
if sk.Key != nil {
return sk, nil
}
return nil, nil
}
// LoadSecretKey retrieves and returns the secret key from bbolt storage, or if no secret key
// was found in storage, it generates, saves, and returns a new secret key.
func (s *storage) LoadSecretKey() (*secretKey, error) {
sk := &secretKey{}
found, err := s.load(skKey, sk, userdataBucket)
if err != nil {
return nil, err
}
if found {
return sk, nil
}
if sk, err = generateSecretKey(); err != nil {
return nil, err
......@@ -192,10 +306,10 @@ func (s *storage) LoadSecretKey() (*secretKey, error) {
return sk, nil
}
func (s *storage) LoadAttributes() (list map[irma.CredentialTypeIdentifier][]*irma.AttributeList, err error) {
func (s *storage) LoadAttributesFile() (list map[irma.CredentialTypeIdentifier][]*irma.AttributeList, err error) {
// The attributes are stored as a list of instances of AttributeList
temp := []*irma.AttributeList{}
if err = s.load(&temp, attributesFile); err != nil {
if err = s.loadFromFile(&temp, attributesFile); err != nil {
return
}
......@@ -216,14 +330,52 @@ func (s *storage) LoadAttributes() (list map[irma.CredentialTypeIdentifier][]*ir
return list, nil
}
func (s *storage) LoadKeyshareServers() (ksses map[irma.SchemeManagerIdentifier]*keyshareServer, err error) {
func (s *storage) LoadAttributes() (list map[irma.CredentialTypeIdentifier][]*irma.AttributeList, err error) {
list = make(map[irma.CredentialTypeIdentifier][]*irma.AttributeList)
return list, s.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(attributesBucket))
if b == nil {
return nil
}
return b.ForEach(func(key, value []byte) error {
var (
credTypeID irma.CredentialTypeIdentifier
attrlistlist []*irma.AttributeList
)
err := json.Unmarshal(key, &credTypeID)
if err != nil {
return err
}
err = json.Unmarshal(value, &attrlistlist)
if err != nil {
return err
}
// Initialize metadata attributes
for _, attrlist := range attrlistlist {
attrlist.MetadataAttribute = irma.MetadataFromInt(attrlist.Ints[0], s.Configuration)
}
list[credTypeID] = attrlistlist
return nil
})
})
}
func (s *storage) LoadKeyshareServersFile() (ksses map[irma.SchemeManagerIdentifier]*keyshareServer, err error) {
ksses = make(map[irma.SchemeManagerIdentifier]*keyshareServer)
if err := s.load(&ksses, kssFile); err != nil {
if err := s.loadFromFile(&ksses, kssFile); err != nil {
return nil, err
}
return ksses, nil
}
func (s *storage) LoadKeyshareServers() (ksses map[irma.SchemeManagerIdentifier]*keyshareServer, err error) {
ksses = make(map[irma.SchemeManagerIdentifier]*keyshareServer)
_, err = s.load(kssKey, &ksses, userdataBucket)
return ksses, err
}
// Returns all logs stored before log with ID 'index' sorted from new to old with
// a maximum result length of 'max'.
func (s *storage) LoadLogsBefore(index uint64, max int) ([]*LogEntry, error) {
......@@ -264,15 +416,27 @@ func (s *storage) loadLogs(max int, startAt func(*bbolt.Cursor) (key, value []by
})
}
func (s *storage) LoadUpdates() (updates []update, err error) {
func (s *storage) LoadUpdatesFile() (updates []update, err error) {
updates = []update{}
if err := s.load(&updates, updatesFile); err != nil {
if err := s.loadFromFile(&updates, updatesFile); err != nil {
return nil, err
}
return updates, nil
}
func (s *storage) LoadUpdates() (updates []update, err error) {
updates = []update{}
_, err = s.load(updatesKey, &updates, userdataBucket)
return
}
func (s *storage) LoadPreferencesFile() (Preferences, error) {
config := defaultPreferences
return config, s.loadFromFile(&config, preferencesFile)
}
func (s *storage) LoadPreferences() (Preferences, error) {
config := defaultPreferences
return config, s.load(&config, preferencesFile)
_, err := s.load(preferencesKey, &config, userdataBucket)
return config, err
}
......@@ -2,6 +2,7 @@ package irmaclient
import (
"encoding/json"
"github.com/privacybydesign/gabi"
"time"
"github.com/privacybydesign/irmago"
......@@ -42,14 +43,11 @@ var clientUpdates = []func(client *Client) error{
// 6: Remove earlier log items of wrong format
nil, // No longer necessary
// 7: Concert log entries to bbolt database
// 7: Convert log entries to bbolt database
func(client *Client) error {
var logs []*LogEntry
var err error
// If loading logs fails, the logs file is likely empty and then there are no log entries to update
// Therefore throw away the error and mark the update as successful.
if err = client.storage.load(&logs, logsFile); err != nil {
if err = client.storage.loadFromFile(&logs, logsFile); err != nil {
return nil
}
// Open one bolt transaction to process all our log entries in
......@@ -74,6 +72,80 @@ var clientUpdates = []func(client *Client) error{
})
return err
},
// 8: Move other user storage to bbolt database
func(client *Client) error {
sk, err := client.storage.LoadSecretKeyFile()
if err != nil {
return err
}
// When no secret key is found, it means the storage is fresh. No update is needed.
if sk == nil {
return nil
}
attrs, err := client.storage.LoadAttributesFile()
if err != nil {
return err
}
sigs := make(map[string]*gabi.CLSignature)
for _, attrlistlist := range attrs {
for _, attrlist := range attrlistlist {
sig, err := client.storage.LoadSignatureFile(attrlist)
if err != nil {
return err
}
sigs[attrlist.Hash()] = sig
}
}
ksses, err := client.storage.LoadKeyshareServersFile()
if err != nil {
return err
}
prefs, err := client.storage.LoadPreferencesFile()
if err != nil {
return err
}
// Preferences are already loaded in client, refresh
client.Preferences = prefs
client.applyPreferences()
updates, err := client.storage.LoadUpdatesFile()
if err != nil {
return err
}
return client.storage.db.Update(func(tx *bbolt.Tx) error {
err = client.storage.TxStoreSecretKey(tx, sk)
if err != nil {
return err
}
err := client.storage.TxStoreAttributes(tx, attrs)
if err != nil {
return err
}
for hash, sig := range sigs {
err = client.storage.TxStoreSignature(tx, hash, sig)
if err != nil {
return err
}
}
err = client.storage.TxStoreKeyshareServers(tx, ksses)
if err != nil {
return err
}
err = client.storage.TxStorePreferences(tx, prefs)
if err != nil {
return err
}
return client.storage.TxStoreUpdates(tx, updates)
})
},
}
// update performs any function from clientUpdates that has not
......@@ -85,6 +157,13 @@ func (client *Client) update() error {
if client.updates, err = client.storage.LoadUpdates(); err != nil {
return err
}
// When no updates are found, it can either be a fresh storage or the storage has not been updated
// to bbolt yet. Therefore also check the updates file.
if len(client.updates) == 0 {
if client.updates, err = client.storage.LoadUpdatesFile(); err != nil {
return err
}
}
// Perform all new updates
for i := len(client.updates); i < len(clientUpdates); i++ {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment