Commit 69048ad0 authored by Ivar Derksen's avatar Ivar Derksen Committed by Sietse Ringers
Browse files

Efficient LoadLogs, bolthold hack removed

parent 04914fc4
...@@ -2,7 +2,6 @@ package sessiontest ...@@ -2,7 +2,6 @@ package sessiontest
import ( import (
"testing" "testing"
"time"
"github.com/privacybydesign/irmago" "github.com/privacybydesign/irmago"
"github.com/privacybydesign/irmago/internal/test" "github.com/privacybydesign/irmago/internal/test"
...@@ -12,7 +11,7 @@ import ( ...@@ -12,7 +11,7 @@ import (
func TestLogging(t *testing.T) { func TestLogging(t *testing.T) {
client, _ := parseStorage(t) client, _ := parseStorage(t)
logs, err := client.LoadLogs(time.Now(), 100) logs, err := client.LoadNewestLogs(100)
oldLogLength := len(logs) oldLogLength := len(logs)
require.NoError(t, err) require.NoError(t, err)
attrid := irma.NewAttributeTypeIdentifier("irma-demo.RU.studentCard.studentID") attrid := irma.NewAttributeTypeIdentifier("irma-demo.RU.studentCard.studentID")
...@@ -22,7 +21,7 @@ func TestLogging(t *testing.T) { ...@@ -22,7 +21,7 @@ func TestLogging(t *testing.T) {
request = getCombinedIssuanceRequest(attrid) request = getCombinedIssuanceRequest(attrid)
sessionHelper(t, request, "issue", client) sessionHelper(t, request, "issue", client)
logs, err = client.LoadLogs(time.Now(), 100) logs, err = client.LoadNewestLogs(100)
require.NoError(t, err) require.NoError(t, err)
require.True(t, len(logs) == oldLogLength+1) require.True(t, len(logs) == oldLogLength+1)
...@@ -36,13 +35,10 @@ func TestLogging(t *testing.T) { ...@@ -36,13 +35,10 @@ func TestLogging(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
require.NotEmpty(t, disclosed) require.NotEmpty(t, disclosed)
// To make sure next session is in a different unix time in seconds to do the before parameter test
time.Sleep(1 * time.Second)
// Do disclosure session // Do disclosure session
request = getDisclosureRequest(attrid) request = getDisclosureRequest(attrid)
sessionHelper(t, request, "verification", client) sessionHelper(t, request, "verification", client)
logs, err = client.LoadLogs(time.Now(), 100) logs, err = client.LoadNewestLogs(100)
require.NoError(t, err) require.NoError(t, err)
require.True(t, len(logs) == oldLogLength+2) require.True(t, len(logs) == oldLogLength+2)
...@@ -54,7 +50,12 @@ func TestLogging(t *testing.T) { ...@@ -54,7 +50,12 @@ func TestLogging(t *testing.T) {
require.NotEmpty(t, disclosed) require.NotEmpty(t, disclosed)
// Test before parameter // Test before parameter
logs, err = client.LoadLogs(time.Time(entry.Time), 100) logs, err = client.LoadLogsBefore(entry.ID, 100)
require.NoError(t, err)
require.True(t, len(logs) == oldLogLength+1)
// Test max parameter
logs, err = client.LoadNewestLogs(1)
require.NoError(t, err) require.NoError(t, err)
require.True(t, len(logs) == oldLogLength+1) require.True(t, len(logs) == oldLogLength+1)
...@@ -63,7 +64,7 @@ func TestLogging(t *testing.T) { ...@@ -63,7 +64,7 @@ func TestLogging(t *testing.T) {
// This issue is fixed, so just run `dep ensure -update github.com/timshannon/bolthold` // This issue is fixed, so just run `dep ensure -update github.com/timshannon/bolthold`
request = getSigningRequest(attrid) request = getSigningRequest(attrid)
sessionHelper(t, request, "signature", client) sessionHelper(t, request, "signature", client)
logs, err = client.LoadLogs(time.Now(), 100) logs, err = client.LoadNewestLogs(100)
require.NoError(t, err) require.NoError(t, err)
require.True(t, len(logs) == oldLogLength+3) require.True(t, len(logs) == oldLogLength+3)
entry = logs[0] entry = logs[0]
......
package irmaclient package irmaclient
import ( import (
"path/filepath"
"strconv" "strconv"
"time" "time"
"path/filepath"
"github.com/bwesterb/go-atum" "github.com/bwesterb/go-atum"
"github.com/getsentry/raven-go" "github.com/getsentry/raven-go"
...@@ -140,7 +140,7 @@ func New( ...@@ -140,7 +140,7 @@ func New(
handler: handler, handler: handler,
} }
cm.Configuration, err = irma.NewConfigurationFromAssets(filepath.Join(storagePath,"irma_configuration"), irmaConfigurationPath) cm.Configuration, err = irma.NewConfigurationFromAssets(filepath.Join(storagePath, "irma_configuration"), irmaConfigurationPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -962,9 +962,16 @@ func (client *Client) KeyshareRemoveAll() error { ...@@ -962,9 +962,16 @@ func (client *Client) KeyshareRemoveAll() error {
// Add, load and store log entries // Add, load and store log entries
// LoadLogs returns the log entries of past events. // LoadNewestLogs returns the log entries of latest past events
func (client *Client) LoadLogs(before time.Time, max int) ([]*LogEntry, error) { // (sorted from new to old, the result length is limited to max).
return client.storage.LoadLogs(before, max) func (client *Client) LoadNewestLogs(max int) ([]*LogEntry, error) {
return client.storage.LoadLogsBefore("", max)
}
// LoadLogsBefore returns the log entries of past events that took place before log entry with ID 'beforeIndex'
// (sorted from new to old, the result length is limited to max).
func (client *Client) LoadLogsBefore(beforeIndex string, max int) ([]*LogEntry, error) {
return client.storage.LoadLogsBefore(beforeIndex, max)
} }
// SetCrashReportingPreference toggles whether or not crash reports should be sent to Sentry. // SetCrashReportingPreference toggles whether or not crash reports should be sent to Sentry.
......
...@@ -12,6 +12,7 @@ import ( ...@@ -12,6 +12,7 @@ import (
// LogEntry is a log entry of a past event. // LogEntry is a log entry of a past event.
type LogEntry struct { type LogEntry struct {
// General info // General info
ID string `boltholdKey:"ID"`
Type irma.Action Type irma.Action
Time irma.Timestamp // Time at which the session was completed Time irma.Timestamp // Time at which the session was completed
......
...@@ -3,9 +3,7 @@ package irmaclient ...@@ -3,9 +3,7 @@ package irmaclient
import ( import (
"encoding/json" "encoding/json"
"io/ioutil" "io/ioutil"
"math"
"os" "os"
"path/filepath" "path/filepath"
"time" "time"
...@@ -125,12 +123,7 @@ func (s *storage) StoreLogs(logs []*LogEntry) error { ...@@ -125,12 +123,7 @@ func (s *storage) StoreLogs(logs []*LogEntry) error {
} }
func (s *storage) AddLogEntry(entry *LogEntry) error { func (s *storage) AddLogEntry(entry *LogEntry) error {
return s.db.Upsert(s.logEntryKey(entry), entry) return s.db.Insert(bolthold.NextSequence(), entry)
}
func (s *storage) logEntryKey(entry *LogEntry) interface{} {
// Time is inverted to reverse sorting order
return math.MaxInt64 ^ time.Time(entry.Time).UnixNano()
} }
func (s *storage) StorePreferences(prefs Preferences) error { func (s *storage) StorePreferences(prefs Preferences) error {
...@@ -206,12 +199,38 @@ func (s *storage) LoadKeyshareServers() (ksses map[irma.SchemeManagerIdentifier] ...@@ -206,12 +199,38 @@ func (s *storage) LoadKeyshareServers() (ksses map[irma.SchemeManagerIdentifier]
return ksses, nil return ksses, nil
} }
func (s *storage) LoadLogs(before time.Time, max int) ([]*LogEntry, error) { // Returns all logs stored before log with ID 'startBeforeIndex' sorted from new to old with a maximum result
// Bolthold key uses inverted times, so before is inverted // length of 'max'. If startBeforeIndex is "". no before is used and the newest logs will be returned.
func (s *storage) LoadLogsBefore(startBeforeIndex string, max int) ([]*LogEntry, error) {
var logs []*LogEntry var logs []*LogEntry
return logs, s.db.Find(&logs, return logs, s.db.Bolt().View(func(tx *bbolt.Tx) error {
bolthold.Where(bolthold.Key).Gt(math.MaxInt64^before.UnixNano()).Limit(max), bucket := tx.Bucket([]byte("LogEntry"))
) if bucket == nil {
return nil
}
c := bucket.Cursor()
var k, v []byte
if startBeforeIndex == "" {
k, v = c.Last()
} else {
c.Seek([]byte(startBeforeIndex))
k, v = c.Prev()
}
for ; k != nil && len(logs) < max; k, v = c.Prev() {
var log LogEntry
if err := json.Unmarshal(v, &log); err != nil {
return err
}
// Manually set ID in struct, because somehow bolthold does not store this (also not when uint64 is used)
log.ID = string(k)
logs = append(logs, &log)
}
return nil
})
} }
func (s *storage) LoadUpdates() (updates []update, err error) { func (s *storage) LoadUpdates() (updates []update, err error) {
......
...@@ -3,6 +3,7 @@ package irmaclient ...@@ -3,6 +3,7 @@ package irmaclient
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/timshannon/bolthold"
"time" "time"
"github.com/privacybydesign/irmago" "github.com/privacybydesign/irmago"
...@@ -60,7 +61,7 @@ var clientUpdates = []func(client *Client) error{ ...@@ -60,7 +61,7 @@ var clientUpdates = []func(client *Client) error{
// Open one bolt transaction to process all our log entries in // Open one bolt transaction to process all our log entries in
err = client.storage.db.Bolt().Update(func(tx *bbolt.Tx) error { err = client.storage.db.Bolt().Update(func(tx *bbolt.Tx) error {
for _, log := range logs { for _, log := range logs {
// As log.Request is a json.RawMessage it woult not get updated to the new session request // As log.Request is a json.RawMessage it would not get updated to the new session request
// format by re-marshaling the containing struct, as normal struct members would, // format by re-marshaling the containing struct, as normal struct members would,
// so update it manually now by marshaling the session request into it. // so update it manually now by marshaling the session request into it.
req, err := log.SessionRequest() req, err := log.SessionRequest()
...@@ -71,7 +72,7 @@ var clientUpdates = []func(client *Client) error{ ...@@ -71,7 +72,7 @@ var clientUpdates = []func(client *Client) error{
if err != nil { if err != nil {
return err return err
} }
if err = client.storage.db.TxUpsert(tx, client.storage.logEntryKey(log), log); err != nil { if err = client.storage.db.TxUpsert(tx, bolthold.NextSequence(), log); err != nil {
return err return err
} }
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment