Commit 452a11ea authored by Sietse Ringers's avatar Sietse Ringers
Browse files

feat: use bolthold for log entries

parent ded9510f
......@@ -2,6 +2,7 @@ package sessiontest
import (
"testing"
"time"
"github.com/privacybydesign/irmago"
"github.com/privacybydesign/irmago/internal/test"
......@@ -11,7 +12,7 @@ import (
func TestLogging(t *testing.T) {
client, _ := parseStorage(t)
logs, err := client.Logs()
logs, err := client.LoadLogs(time.Now(), 100)
oldLogLength := len(logs)
require.NoError(t, err)
attrid := irma.NewAttributeTypeIdentifier("irma-demo.RU.studentCard.studentID")
......@@ -21,7 +22,7 @@ func TestLogging(t *testing.T) {
request = getCombinedIssuanceRequest(attrid)
sessionHelper(t, request, "issue", client)
logs, err = client.Logs()
logs, err = client.LoadLogs(time.Now(), 100)
require.NoError(t, err)
require.True(t, len(logs) == oldLogLength+1)
......@@ -38,7 +39,7 @@ func TestLogging(t *testing.T) {
// Do disclosure session
request = getDisclosureRequest(attrid)
sessionHelper(t, request, "verification", client)
logs, err = client.Logs()
logs, err = client.LoadLogs(time.Now(), 100)
require.NoError(t, err)
require.True(t, len(logs) == oldLogLength+2)
......@@ -52,7 +53,7 @@ func TestLogging(t *testing.T) {
// Do signature session
request = getSigningRequest(attrid)
sessionHelper(t, request, "signature", client)
logs, err = client.Logs()
logs, err = client.LoadLogs(time.Now(), 100)
require.NoError(t, err)
require.True(t, len(logs) == oldLogLength+3)
entry = logs[len(logs)-1]
......
......@@ -44,7 +44,6 @@ type Client struct {
attributes map[irma.CredentialTypeIdentifier][]*irma.AttributeList
credentialsCache map[irma.CredentialTypeIdentifier]map[int]*credential
keyshareServers map[irma.SchemeManagerIdentifier]*keyshareServer
logs []*LogEntry
updates []update
// Where we store/load it to/from
......@@ -289,7 +288,7 @@ func (client *Client) remove(id irma.CredentialTypeIdentifier, index int, storen
removed[id] = attrs.Strings()
if storenow {
return client.addLogEntry(&LogEntry{
return client.storage.AddLogEntry(&LogEntry{
Type: actionRemoval,
Time: irma.Timestamp(time.Now()),
Removed: removed,
......@@ -333,10 +332,7 @@ func (client *Client) RemoveAllCredentials() error {
Time: irma.Timestamp(time.Now()),
Removed: removed,
}
if err := client.addLogEntry(logentry); err != nil {
return err
}
return client.storage.StoreLogs(client.logs)
return client.storage.AddLogEntry(logentry)
}
// Attribute and credential getter methods
......@@ -966,21 +962,9 @@ func (client *Client) KeyshareRemoveAll() error {
// Add, load and store log entries
func (client *Client) addLogEntry(entry *LogEntry) error {
client.logs = append(client.logs, entry)
return client.storage.StoreLogs(client.logs)
}
// Logs returns the log entries of past events.
func (client *Client) Logs() ([]*LogEntry, error) {
if client.logs == nil || len(client.logs) == 0 {
var err error
client.logs, err = client.storage.LoadLogs()
if err != nil {
return nil, err
}
}
return client.logs, nil
// LoadLogs returns the log entries of past events.
func (client *Client) LoadLogs(before time.Time, max int) ([]*LogEntry, error) {
return client.storage.LoadLogs(before, max)
}
// SetCrashReportingPreference toggles whether or not crash reports should be sent to Sentry.
......
......@@ -14,25 +14,32 @@ type LogEntry struct {
// General info
Type irma.Action
Time irma.Timestamp // Time at which the session was completed
Version *irma.ProtocolVersion `json:",omitempty"` // Protocol version that was used in the session
Request json.RawMessage `json:",omitempty"` // Message that started the session
request irma.SessionRequest // cached parsed version of Request; get with LogEntry.SessionRequest()
// Credential removal
Removed map[irma.CredentialTypeIdentifier][]irma.TranslatedString `json:",omitempty"`
// Session type-specific info
Removed map[irma.CredentialTypeIdentifier][]irma.TranslatedString `json:",omitempty"` // In case of credential removal
SignedMessage []byte `json:",omitempty"` // In case of signature sessions
Timestamp *atum.Timestamp `json:",omitempty"` // In case of signature sessions
SignedMessageLDContext string `json:",omitempty"` // In case of signature sessions
// Signature sessions
SignedMessage []byte `json:",omitempty"`
Timestamp *atum.Timestamp `json:",omitempty"`
SignedMessageLDContext string `json:",omitempty"`
// Issuance sessions
IssueCommitment *irma.IssueCommitmentMessage `json:",omitempty"`
// All session types
Version *irma.ProtocolVersion `json:",omitempty"`
Disclosure *irma.Disclosure `json:",omitempty"`
Request json.RawMessage `json:",omitempty"` // Message that started the session
request irma.SessionRequest // cached parsed version of Request; get with LogEntry.SessionRequest()
}
const actionRemoval = irma.Action("removal")
func (entry *LogEntry) SessionRequest() (irma.SessionRequest, error) {
if entry.request == nil {
if entry.request != nil {
return entry.request, nil
}
switch entry.Type {
case irma.ActionDisclosing:
entry.request = &irma.DisclosureRequest{}
......@@ -43,7 +50,6 @@ func (entry *LogEntry) SessionRequest() (irma.SessionRequest, error) {
default:
return nil, nil
}
}
err := json.Unmarshal([]byte(entry.Request), entry.request)
if err != nil {
......@@ -132,8 +138,7 @@ func (session *session) createLogEntry(response interface{}) (*LogEntry, error)
case irma.ActionSigning:
// Get the signed message and timestamp
request := session.request.(*irma.SignatureRequest)
entry.SignedMessage = []byte(request.Message)
entry.SignedMessage = []byte(session.request.(*irma.SignatureRequest).Message)
entry.Timestamp = session.timestamp
entry.SignedMessageLDContext = irma.LDContextSignedMessage
......
......@@ -395,7 +395,10 @@ func (session *session) sendResponse(message interface{}) {
return
}
}
log, _ = session.createLogEntry(message) // TODO err
log, err = session.createLogEntry(message)
if err != nil {
irma.Logger.Warn(errors.WrapPrefix(err, "Failed to create log entry", 0).ErrorStack())
}
case irma.ActionDisclosing:
messageJson, err = json.Marshal(message)
if err != nil {
......@@ -413,7 +416,10 @@ func (session *session) sendResponse(message interface{}) {
return
}
}
log, _ = session.createLogEntry(message) // TODO err
log, err = session.createLogEntry(message)
if err != nil {
irma.Logger.Warn(errors.WrapPrefix(err, "Failed to create log entry", 0).ErrorStack())
}
case irma.ActionIssuing:
response := []*gabi.IssueSignatureMessage{}
if err = session.transport.Post("commitments", &response, message); err != nil {
......@@ -424,10 +430,15 @@ func (session *session) sendResponse(message interface{}) {
session.fail(&irma.SessionError{ErrorType: irma.ErrorCrypto, Err: err})
return
}
log, _ = session.createLogEntry(message) // TODO err
log, err = session.createLogEntry(message)
if err != nil {
irma.Logger.Warn(errors.WrapPrefix(err, "Failed to create log entry", 0).ErrorStack())
}
}
_ = session.client.addLogEntry(log) // TODO err
if err = session.client.storage.AddLogEntry(log); err != nil {
irma.Logger.Warn(errors.WrapPrefix(err, "Failed to write log entry", 0).ErrorStack())
}
if session.Action == irma.ActionIssuing {
session.client.handler.UpdateAttributes()
}
......
......@@ -4,11 +4,15 @@ import (
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"time"
"github.com/privacybydesign/gabi"
"github.com/privacybydesign/irmago"
"github.com/privacybydesign/irmago/internal/fs"
"github.com/timshannon/bolthold"
"go.etcd.io/bbolt"
)
// This file contains the storage struct and its methods,
......@@ -17,6 +21,7 @@ import (
// Storage provider for a Client
type storage struct {
storagePath string
db *bolthold.Store
Configuration *irma.Configuration
}
......@@ -29,6 +34,8 @@ const (
logsFile = "logs"
preferencesFile = "preferences"
signaturesDir = "sigs"
databaseFile = "db"
)
func (s *storage) path(p string) string {
......@@ -41,10 +48,19 @@ func (s *storage) path(p string) string {
// Setting it up in a properly protected location (e.g., with automatic
// backups to iCloud/Google disabled) is the responsibility of the user.
func (s *storage) EnsureStorageExists() error {
if err := fs.AssertPathExists(s.storagePath); err != nil {
var err error
if err = fs.AssertPathExists(s.storagePath); err != nil {
return err
}
return fs.EnsureDirectoryExists(s.path(signaturesDir))
if err = fs.EnsureDirectoryExists(s.path(signaturesDir)); err != nil {
return err
}
s.db, err = bolthold.Open(s.path(databaseFile), 0600, &bolthold.Options{
Options: &bbolt.Options{Timeout: 1 * time.Second},
Encoder: json.Marshal,
Decoder: json.Unmarshal,
})
return err
}
func (s *storage) load(dest interface{}, path string) (err error) {
......@@ -107,6 +123,14 @@ func (s *storage) StoreLogs(logs []*LogEntry) error {
return s.store(logs, logsFile)
}
func (s *storage) AddLogEntry(entry *LogEntry) error {
return s.db.Upsert(s.logEntryKey(entry), entry)
}
func (s *storage) logEntryKey(entry *LogEntry) interface{} {
return time.Time(entry.Time).UnixNano()
}
func (s *storage) StorePreferences(prefs Preferences) error {
return s.store(prefs, preferencesFile)
}
......@@ -180,12 +204,11 @@ func (s *storage) LoadKeyshareServers() (ksses map[irma.SchemeManagerIdentifier]
return ksses, nil
}
func (s *storage) LoadLogs() (logs []*LogEntry, err error) {
logs = []*LogEntry{}
if err := s.load(&logs, logsFile); err != nil {
return nil, err
}
return logs, nil
func (s *storage) LoadLogs(before time.Time, max int) ([]*LogEntry, error) {
var logs []*LogEntry
return logs, s.db.Find(&logs,
bolthold.Where(bolthold.Key).Lt(before.UnixNano()).Limit(max),
)
}
func (s *storage) LoadUpdates() (updates []update, err error) {
......
package irmaclient
import (
"encoding/json"
"fmt"
"time"
"github.com/privacybydesign/irmago"
"github.com/privacybydesign/irmago/internal/fs"
"go.etcd.io/bbolt"
)
// This file contains the update mechanism for Client
......@@ -27,46 +29,59 @@ var clientUpdates = []func(client *Client) error{
nil, // made irrelevant by irma_configuration-autocopying
// 2: Rename config -> preferences
func(client *Client) (err error) {
exists, err := fs.PathExists(client.storage.path("config"))
if !exists || err != nil {
return
}
oldStruct := &struct {
SendCrashReports bool
}{}
// Load old file, convert to new struct, and save
err = client.storage.load(oldStruct, "config")
if err != nil {
return err
}
client.Preferences = Preferences{
EnableCrashReporting: oldStruct.SendCrashReports,
}
return client.storage.StorePreferences(client.Preferences)
},
nil, // No longer necessary
// 3: Copy new irma_configuration out of assets
nil, // made irrelevant by irma_configuration-autocopying
// 4: For each keyshare server, include in its struct the identifier of its scheme manager
func(client *Client) (err error) {
keyshareServers, err := client.storage.LoadKeyshareServers()
nil, // No longer necessary
// 5: Remove the test scheme manager which was erroneously included in a production build
nil, // No longer necessary, also broke many unit tests
// 6: Remove earlier log items of wrong format
nil, // No longer necessary
// 7: Concert log entries to bolthold database
func(client *Client) error {
var logs []*LogEntry
var err error
start := time.Now()
fmt.Println(start.UnixNano(), "started")
if err = client.storage.load(&logs, logsFile); err != nil {
return err
}
loaded := time.Now()
fmt.Println(loaded.Sub(start), "loaded")
// Open one bolt transaction to process all our log entries in
err = client.storage.db.Bolt().Update(func(tx *bbolt.Tx) error {
for _, log := range logs {
// As log.Request is a json.RawMessage it woult not get updated to the new session request
// format by re-marshaling the containing struct, as normal struct members would,
// so update it manually now by marshaling the session request into it.
req, err := log.SessionRequest()
if err != nil {
return err
}
for smi, kss := range keyshareServers {
kss.SchemeManagerIdentifier = smi
log.Request, err = json.Marshal(req)
if err != nil {
return err
}
return client.storage.StoreKeyshareServers(keyshareServers)
},
if err = client.storage.db.TxUpsert(tx, client.storage.logEntryKey(log), log); err != nil {
return err
}
}
return nil
})
// 5: Remove the test scheme manager which was erroneously included in a production build
nil, // No longer necessary, also broke many unit tests
done := time.Now()
fmt.Println(done.Sub(loaded), "done converting", len(logs), "logs")
// 6: Remove earlier log items of wrong format
func(client *Client) (err error) {
return client.storage.StoreLogs([]*LogEntry{})
return err
},
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment