From 81ab0cccfcca3f9916d1d25eca9350c01ec7b2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ba=CC=88uerlein?= Date: Sat, 16 Jun 2018 13:45:37 +0200 Subject: [PATCH 01/18] Use custom interface for database instead of generic one In order to support the generic helper functions for transactions and usual connections, we used to use the generic interface and do manual type-checks and conversions. However, as sqlx.DB and sqlx.Tx share the same method signatures, I created a new interface that contains all used methods and was able to replace the usage of the generic interface{} with the new one. --- controllers/effects/effects.go | 3 +- controllers/stats/stats.go | 3 +- models/asset_stat/asset_stat.go | 6 +-- models/cursor/cursor.go | 4 +- models/effect/effect.go | 4 +- utils/sql/sql.go | 69 +++++++++------------------------ 6 files changed, 29 insertions(+), 60 deletions(-) diff --git a/controllers/effects/effects.go b/controllers/effects/effects.go index 310fdfc..9382768 100644 --- a/controllers/effects/effects.go +++ b/controllers/effects/effects.go @@ -3,12 +3,13 @@ package effects import ( "github.com/cndy-store/analytics/models/effect" "github.com/cndy-store/analytics/utils/filter" + "github.com/cndy-store/analytics/utils/sql" "github.com/gin-gonic/gin" "log" "net/http" ) -func Init(db interface{}, router *gin.Engine) { +func Init(db sql.Database, router *gin.Engine) { // GET /effects[?from=XXX&to=XXX] router.GET("/effects", func(c *gin.Context) { from, to, err := filter.Parse(c) diff --git a/controllers/stats/stats.go b/controllers/stats/stats.go index cdb977b..fca26d3 100644 --- a/controllers/stats/stats.go +++ b/controllers/stats/stats.go @@ -4,12 +4,13 @@ import ( "github.com/cndy-store/analytics/models/asset_stat" "github.com/cndy-store/analytics/models/cursor" "github.com/cndy-store/analytics/utils/filter" + "github.com/cndy-store/analytics/utils/sql" "github.com/gin-gonic/gin" "log" "net/http" ) -func Init(db interface{}, router *gin.Engine) { +func Init(db sql.Database, router *gin.Engine) { // GET /stats[?from=XXX&to=XXX] router.GET("/stats", func(c *gin.Context) { from, to, err := filter.Parse(c) diff --git a/models/asset_stat/asset_stat.go b/models/asset_stat/asset_stat.go index abc41cc..17ccd77 100644 --- a/models/asset_stat/asset_stat.go +++ b/models/asset_stat/asset_stat.go @@ -29,7 +29,7 @@ type AssetStat struct { JsonTransferred *string `db:"-" json:"transferred"` } -func New(db interface{}, effect horizon.Effect, timestamp time.Time) (err error) { +func New(db sql.Database, effect horizon.Effect, timestamp time.Time) (err error) { // Store amount_transfered and amount_issued upon insert in a different table // (analogue to the asset endpoint of Horizon) @@ -63,7 +63,7 @@ func (f *Filter) Defaults() { } } -func Get(db interface{}, filter Filter) (stats []AssetStat, err error) { +func Get(db sql.Database, filter Filter) (stats []AssetStat, err error) { filter.Defaults() err = sql.Select(db, &stats, `SELECT * FROM asset_stats WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY id`, filter.From, filter.To) @@ -79,7 +79,7 @@ func Get(db interface{}, filter Filter) (stats []AssetStat, err error) { return } -func Latest(db interface{}) (stats AssetStat, err error) { +func Latest(db sql.Database) (stats AssetStat, err error) { err = sql.Get(db, &stats, `SELECT * FROM asset_stats ORDER BY id DESC LIMIT 1`) if err == sql.ErrNoRows { log.Printf("[ERROR] asset_stat.Latest(): %s", err) diff --git a/models/cursor/cursor.go b/models/cursor/cursor.go index 11577f9..5ffd61a 100644 --- a/models/cursor/cursor.go +++ b/models/cursor/cursor.go @@ -18,12 +18,12 @@ func Update(cursor horizon.Cursor) { Current = cursor } -func Save(db interface{}) (err error) { +func Save(db sql.Database) (err error) { _, err = sql.Exec(db, `UPDATE cursors SET paging_token=$1 WHERE id=1`, Current) return } -func LoadLatest(db interface{}) (err error) { +func LoadLatest(db sql.Database) (err error) { var c string err = sql.Get(db, &c, `SELECT paging_token FROM cursors WHERE id=1`) if err != nil { diff --git a/models/effect/effect.go b/models/effect/effect.go index ccffbb8..9c78f2d 100644 --- a/models/effect/effect.go +++ b/models/effect/effect.go @@ -49,7 +49,7 @@ type Operation struct { CreatedAt time.Time `json:"created_at"` } -func New(db interface{}, effect horizon.Effect) (err error) { +func New(db sql.Database, effect horizon.Effect) (err error) { // Get operation operation := getOperation(effect.Links.Operation.Href) @@ -142,7 +142,7 @@ func (f *Filter) Defaults() { } } -func Get(db interface{}, filter Filter) (effects []Effect, err error) { +func Get(db sql.Database, filter Filter) (effects []Effect, err error) { filter.Defaults() err = sql.Select(db, &effects, `SELECT * FROM effects WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY created_at`, filter.From, filter.To) diff --git a/utils/sql/sql.go b/utils/sql/sql.go index 4dbebd9..c4938c6 100644 --- a/utils/sql/sql.go +++ b/utils/sql/sql.go @@ -2,7 +2,6 @@ package sql import ( "database/sql" - "errors" "github.com/golang-migrate/migrate" "github.com/golang-migrate/migrate/database/postgres" _ "github.com/golang-migrate/migrate/source/file" @@ -12,6 +11,14 @@ import ( var ErrNoRows = sql.ErrNoRows +type Database interface { + Exec(query string, args ...interface{}) (sql.Result, error) + Get(dest interface{}, query string, args ...interface{}) error + Select(dest interface{}, query string, args ...interface{}) error + PrepareNamed(query string) (*sqlx.NamedStmt, error) + QueryRow(query string, args ...interface{}) *sql.Row +} + // Open production database and run migrations func OpenAndMigrate(relPath string) (db *sqlx.DB, err error) { /* Open connection to postgresql to the server specified in the following environment variables: @@ -45,56 +52,24 @@ func OpenAndMigrate(relPath string) (db *sqlx.DB, err error) { } // Exec is a type agnostic wrapper for sqlx.Exec() (works with sqlx.DB and sqlx.Tx) -func Exec(db interface{}, query string, args ...interface{}) (result sql.Result, err error) { - switch db.(type) { - case *sqlx.DB: - result, err = db.(*sqlx.DB).Exec(query, args...) - case *sqlx.Tx: - result, err = db.(*sqlx.Tx).Exec(query, args...) - default: - err = errors.New("Unknown DB interface{} in sql.Exec()") - } - return +func Exec(db Database, query string, args ...interface{}) (result sql.Result, err error) { + return db.Exec(query, args...) } // Getis a type agnostic wrapper for sqlx.Get() (works with sqlx.DB and sqlx.Tx) -func Get(db, obj interface{}, query string, args ...interface{}) (err error) { - switch db.(type) { - case *sqlx.DB: - err = db.(*sqlx.DB).Get(obj, query, args...) - case *sqlx.Tx: - err = db.(*sqlx.Tx).Get(obj, query, args...) - default: - err = errors.New("Unknown DB interface{} in sql.Get()") - } - return +func Get(db Database, obj interface{}, query string, args ...interface{}) (err error) { + return db.Get(obj, query, args...) } // Select is a type agnostic wrapper for sqlx.Select() (works with sqlx.DB and sqlx.Tx) -func Select(db interface{}, obj interface{}, query string, args ...interface{}) (err error) { - switch db.(type) { - case *sqlx.DB: - err = db.(*sqlx.DB).Select(obj, query, args...) - case *sqlx.Tx: - err = db.(*sqlx.Tx).Select(obj, query, args...) - default: - err = errors.New("Unknown DB interface{} in sql.Select()") - } - return +func Select(db Database, obj interface{}, query string, args ...interface{}) (err error) { + return db.Select(obj, query, args...) } // NamedQuery is a type agnostic wrapper for sqlx.NamedQuery() (works with sqlx.DB and sqlx.Tx) -func NamedQuery(db interface{}, obj interface{}, query string, arg interface{}) (err error) { +func NamedQuery(db Database, obj interface{}, query string, arg interface{}) (err error) { var stmt *sqlx.NamedStmt - - switch db.(type) { - case *sqlx.DB: - stmt, err = db.(*sqlx.DB).PrepareNamed(query) - case *sqlx.Tx: - stmt, err = db.(*sqlx.Tx).PrepareNamed(query) - default: - err = errors.New("Unknown DB interface{} in sql.NamedQuery()") - } + stmt, err = db.PrepareNamed(query) if err != nil { return @@ -110,20 +85,12 @@ func NamedQuery(db interface{}, obj interface{}, query string, arg interface{}) } // Exists is a type agnostic function that checks whether a statement returns a row -func Exists(db interface{}, query string, args ...interface{}) (exists bool, err error) { +func Exists(db Database, query string, args ...interface{}) (exists bool, err error) { var row *sql.Row // Prepare exists query query = `SELECT EXISTS(` + query + `) LIMIT 1` - - switch db.(type) { - case *sqlx.DB: - row = db.(*sqlx.DB).QueryRow(query, args...) - case *sqlx.Tx: - row = db.(*sqlx.Tx).QueryRow(query, args...) - default: - err = errors.New("Unknown DB interface{} in sql.Exists()") - } + row = db.QueryRow(query, args...) err = row.Scan(&exists) return From 3b241a286744e738c70c53095353e37538684071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ba=CC=88uerlein?= Date: Sat, 16 Jun 2018 13:54:26 +0200 Subject: [PATCH 02/18] Remove uneccessary Exec helper --- models/asset_stat/asset_stat.go | 2 +- models/cursor/cursor.go | 2 +- models/effect/effect.go | 2 +- utils/sql/sql.go | 5 ----- utils/test/test.go | 5 ++--- 5 files changed, 5 insertions(+), 11 deletions(-) diff --git a/models/asset_stat/asset_stat.go b/models/asset_stat/asset_stat.go index 17ccd77..e8e406c 100644 --- a/models/asset_stat/asset_stat.go +++ b/models/asset_stat/asset_stat.go @@ -33,7 +33,7 @@ func New(db sql.Database, effect horizon.Effect, timestamp time.Time) (err error // Store amount_transfered and amount_issued upon insert in a different table // (analogue to the asset endpoint of Horizon) - _, err = sql.Exec(db, `INSERT INTO asset_stats(paging_token, asset_code, asset_issuer, asset_type, created_at, issued, transferred, accounts_with_trustline, accounts_with_payments, payments) + _, err = db.Exec(`INSERT INTO asset_stats(paging_token, asset_code, asset_issuer, asset_type, created_at, issued, transferred, accounts_with_trustline, accounts_with_payments, payments) VALUES ($1, $2, $3, $4, $5, (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type='account_debited' AND account=$6), (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type='account_debited' AND account!=$6), diff --git a/models/cursor/cursor.go b/models/cursor/cursor.go index 5ffd61a..eb2890b 100644 --- a/models/cursor/cursor.go +++ b/models/cursor/cursor.go @@ -19,7 +19,7 @@ func Update(cursor horizon.Cursor) { } func Save(db sql.Database) (err error) { - _, err = sql.Exec(db, `UPDATE cursors SET paging_token=$1 WHERE id=1`, Current) + _, err = db.Exec(`UPDATE cursors SET paging_token=$1 WHERE id=1`, Current) return } diff --git a/models/effect/effect.go b/models/effect/effect.go index 9c78f2d..3840667 100644 --- a/models/effect/effect.go +++ b/models/effect/effect.go @@ -80,7 +80,7 @@ func New(db sql.Database, effect horizon.Effect) (err error) { } // Just input the fields we're requiring for now, can be replayed anytime form the chain later. - _, err = sql.Exec(db, `INSERT INTO effects( + _, err = db.Exec(`INSERT INTO effects( effect_id, operation, succeeds, precedes, paging_token, account, amount, type, type_i, starting_balance, diff --git a/utils/sql/sql.go b/utils/sql/sql.go index c4938c6..a991368 100644 --- a/utils/sql/sql.go +++ b/utils/sql/sql.go @@ -51,11 +51,6 @@ func OpenAndMigrate(relPath string) (db *sqlx.DB, err error) { return } -// Exec is a type agnostic wrapper for sqlx.Exec() (works with sqlx.DB and sqlx.Tx) -func Exec(db Database, query string, args ...interface{}) (result sql.Result, err error) { - return db.Exec(query, args...) -} - // Getis a type agnostic wrapper for sqlx.Get() (works with sqlx.DB and sqlx.Tx) func Get(db Database, obj interface{}, query string, args ...interface{}) (err error) { return db.Get(obj, query, args...) diff --git a/utils/test/test.go b/utils/test/test.go index 7c98796..515c477 100644 --- a/utils/test/test.go +++ b/utils/test/test.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/cndy-store/analytics/utils/bigint" "github.com/cndy-store/analytics/utils/cndy" - "github.com/cndy-store/analytics/utils/sql" "github.com/jmoiron/sqlx" "time" ) @@ -50,7 +49,7 @@ func InsertTestData(tx *sqlx.Tx) (err error) { return } - _, err = sql.Exec(tx, `INSERT INTO effects(effect_id, operation, paging_token, account, amount, type, asset_type, asset_issuer, asset_code, created_at) + _, err = tx.Exec(`INSERT INTO effects(effect_id, operation, paging_token, account, amount, type, asset_type, asset_issuer, asset_code, created_at) VALUES($1, 'https://horizon-testnet.stellar.org/operations/34028708058632193', $2, $3, $4, $5, 'credit_alphanum4', $6, $7, $8)`, fmt.Sprintf("0034028708058632193-%09d", i), data.PagingToken, data.Account, amount, data.Type, cndy.AssetIssuer, cndy.AssetCode, data.CreatedAt) if err != nil { @@ -58,7 +57,7 @@ func InsertTestData(tx *sqlx.Tx) (err error) { } } - _, err = sql.Exec(tx, `SELECT repopulate_asset_stats()`) + _, err = tx.Exec(`SELECT repopulate_asset_stats()`) if err != nil { return } From 5264001e6ed995d4514fd85f7f81f6c554caeb59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ba=CC=88uerlein?= Date: Sat, 16 Jun 2018 13:55:57 +0200 Subject: [PATCH 03/18] Remove unnecessary Get helper --- models/asset_stat/asset_stat.go | 2 +- models/cursor/cursor.go | 2 +- utils/sql/sql.go | 5 ----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/models/asset_stat/asset_stat.go b/models/asset_stat/asset_stat.go index e8e406c..c25bc2d 100644 --- a/models/asset_stat/asset_stat.go +++ b/models/asset_stat/asset_stat.go @@ -80,7 +80,7 @@ func Get(db sql.Database, filter Filter) (stats []AssetStat, err error) { } func Latest(db sql.Database) (stats AssetStat, err error) { - err = sql.Get(db, &stats, `SELECT * FROM asset_stats ORDER BY id DESC LIMIT 1`) + err = db.Get(&stats, `SELECT * FROM asset_stats ORDER BY id DESC LIMIT 1`) if err == sql.ErrNoRows { log.Printf("[ERROR] asset_stat.Latest(): %s", err) } diff --git a/models/cursor/cursor.go b/models/cursor/cursor.go index eb2890b..9a9c464 100644 --- a/models/cursor/cursor.go +++ b/models/cursor/cursor.go @@ -25,7 +25,7 @@ func Save(db sql.Database) (err error) { func LoadLatest(db sql.Database) (err error) { var c string - err = sql.Get(db, &c, `SELECT paging_token FROM cursors WHERE id=1`) + err = db.Get(&c, `SELECT paging_token FROM cursors WHERE id=1`) if err != nil { return } diff --git a/utils/sql/sql.go b/utils/sql/sql.go index a991368..af56b4f 100644 --- a/utils/sql/sql.go +++ b/utils/sql/sql.go @@ -51,11 +51,6 @@ func OpenAndMigrate(relPath string) (db *sqlx.DB, err error) { return } -// Getis a type agnostic wrapper for sqlx.Get() (works with sqlx.DB and sqlx.Tx) -func Get(db Database, obj interface{}, query string, args ...interface{}) (err error) { - return db.Get(obj, query, args...) -} - // Select is a type agnostic wrapper for sqlx.Select() (works with sqlx.DB and sqlx.Tx) func Select(db Database, obj interface{}, query string, args ...interface{}) (err error) { return db.Select(obj, query, args...) From f325dcca7f45054f950f12795d86d9b11e269db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ba=CC=88uerlein?= Date: Sat, 16 Jun 2018 13:57:02 +0200 Subject: [PATCH 04/18] Remove unnecessary Select helper --- models/asset_stat/asset_stat.go | 2 +- models/effect/effect.go | 2 +- utils/sql/sql.go | 5 ----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/models/asset_stat/asset_stat.go b/models/asset_stat/asset_stat.go index c25bc2d..75c7fa8 100644 --- a/models/asset_stat/asset_stat.go +++ b/models/asset_stat/asset_stat.go @@ -65,7 +65,7 @@ func (f *Filter) Defaults() { func Get(db sql.Database, filter Filter) (stats []AssetStat, err error) { filter.Defaults() - err = sql.Select(db, &stats, `SELECT * FROM asset_stats WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY id`, + err = db.Select(&stats, `SELECT * FROM asset_stats WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY id`, filter.From, filter.To) if err == sql.ErrNoRows { log.Printf("[ERROR] asset_stat.Get(): %s", err) diff --git a/models/effect/effect.go b/models/effect/effect.go index 3840667..8960a44 100644 --- a/models/effect/effect.go +++ b/models/effect/effect.go @@ -144,7 +144,7 @@ func (f *Filter) Defaults() { func Get(db sql.Database, filter Filter) (effects []Effect, err error) { filter.Defaults() - err = sql.Select(db, &effects, `SELECT * FROM effects WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY created_at`, + err = db.Select(&effects, `SELECT * FROM effects WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY created_at`, filter.From, filter.To) if err == sql.ErrNoRows { log.Printf("[ERROR] effect.Get(): %s", err) diff --git a/utils/sql/sql.go b/utils/sql/sql.go index af56b4f..b4c7570 100644 --- a/utils/sql/sql.go +++ b/utils/sql/sql.go @@ -51,11 +51,6 @@ func OpenAndMigrate(relPath string) (db *sqlx.DB, err error) { return } -// Select is a type agnostic wrapper for sqlx.Select() (works with sqlx.DB and sqlx.Tx) -func Select(db Database, obj interface{}, query string, args ...interface{}) (err error) { - return db.Select(obj, query, args...) -} - // NamedQuery is a type agnostic wrapper for sqlx.NamedQuery() (works with sqlx.DB and sqlx.Tx) func NamedQuery(db Database, obj interface{}, query string, arg interface{}) (err error) { var stmt *sqlx.NamedStmt From b4dbe209b8bc6c0edc93a58dc3fc76a6b3b7d9ee Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Sat, 16 Jun 2018 14:30:24 +0200 Subject: [PATCH 05/18] Add missing SQL functions to utils/sql --- utils/sql/sql.go | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/utils/sql/sql.go b/utils/sql/sql.go index b4c7570..5905202 100644 --- a/utils/sql/sql.go +++ b/utils/sql/sql.go @@ -1,6 +1,7 @@ package sql import ( + "context" "database/sql" "github.com/golang-migrate/migrate" "github.com/golang-migrate/migrate/database/postgres" @@ -11,12 +12,40 @@ import ( var ErrNoRows = sql.ErrNoRows +// This type serves as an abstraction for sqlx.DB and sqlx.Tx and supports all functions both +// of the types have in common. This allows as to use their functions type agnostically. type Database interface { + // Common functions of sql.DB and sql.Tx Exec(query string, args ...interface{}) (sql.Result, error) + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) + Prepare(query string) (*sql.Stmt, error) + PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) + Query(query string, args ...interface{}) (*sql.Rows, error) + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) + QueryRow(query string, args ...interface{}) *sql.Row + QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row + + // Common functions of sqlx.DB and sqlx.Tx + BindNamed(query string, arg interface{}) (string, []interface{}, error) + DriverName() string Get(dest interface{}, query string, args ...interface{}) error - Select(dest interface{}, query string, args ...interface{}) error + GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error + MustExec(query string, args ...interface{}) sql.Result + MustExecContext(ctx context.Context, query string, args ...interface{}) sql.Result + NamedExec(query string, arg interface{}) (sql.Result, error) + NamedExecContext(ctx context.Context, query string, arg interface{}) (sql.Result, error) + NamedQuery(query string, arg interface{}) (*sqlx.Rows, error) PrepareNamed(query string) (*sqlx.NamedStmt, error) - QueryRow(query string, args ...interface{}) *sql.Row + PrepareNamedContext(ctx context.Context, query string) (*sqlx.NamedStmt, error) + Preparex(query string) (*sqlx.Stmt, error) + PreparexContext(ctx context.Context, query string) (*sqlx.Stmt, error) + QueryRowx(query string, args ...interface{}) *sqlx.Row + QueryRowxContext(ctx context.Context, query string, args ...interface{}) *sqlx.Row + Queryx(query string, args ...interface{}) (*sqlx.Rows, error) + QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) + Rebind(query string) string + Select(dest interface{}, query string, args ...interface{}) error + SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error } // Open production database and run migrations From 7ab84a31b98e272da6f7a80c225fb9f930986f5e Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Sun, 17 Jun 2018 15:19:57 +0200 Subject: [PATCH 06/18] Remove ancient comment --- models/effect/effect.go | 1 - 1 file changed, 1 deletion(-) diff --git a/models/effect/effect.go b/models/effect/effect.go index 8960a44..9ec8c96 100644 --- a/models/effect/effect.go +++ b/models/effect/effect.go @@ -79,7 +79,6 @@ func New(db sql.Database, effect horizon.Effect) (err error) { return } - // Just input the fields we're requiring for now, can be replayed anytime form the chain later. _, err = db.Exec(`INSERT INTO effects( effect_id, operation, succeeds, precedes, From a9b7b86098f1eb261597832a3fbd7cd69057948a Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Sun, 17 Jun 2018 18:03:26 +0200 Subject: [PATCH 07/18] Use JSON resonses consistently --- controllers/effects/effects.go | 6 +++++- controllers/stats/stats.go | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/controllers/effects/effects.go b/controllers/effects/effects.go index 9382768..c9c7c01 100644 --- a/controllers/effects/effects.go +++ b/controllers/effects/effects.go @@ -25,11 +25,15 @@ func Init(db sql.Database, router *gin.Engine) { effects, err := effect.Get(db, effect.Filter{From: from, To: to}) if err != nil { log.Printf("[ERROR] Couldn't get effect from database: %s", err) - c.String(http.StatusInternalServerError, "") + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) return } c.JSON(http.StatusOK, gin.H{ + "status": "ok", "effects": effects, }) return diff --git a/controllers/stats/stats.go b/controllers/stats/stats.go index fca26d3..c8d2b16 100644 --- a/controllers/stats/stats.go +++ b/controllers/stats/stats.go @@ -26,12 +26,16 @@ func Init(db sql.Database, router *gin.Engine) { assetStats, err := assetStat.Get(db, assetStat.Filter{From: from, To: to}) if err != nil { log.Printf("[ERROR] Couldn't get asset stats from database: %s", err) - c.String(http.StatusInternalServerError, "") + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) return } c.JSON(http.StatusOK, gin.H{ - "stats": assetStats, + "status": "ok", + "stats": assetStats, }) return }) @@ -40,11 +44,15 @@ func Init(db sql.Database, router *gin.Engine) { latest, err := assetStat.Latest(db) if err != nil { log.Printf("[ERROR] Couldn't get asset stats from database: %s", err) - c.String(http.StatusInternalServerError, "") + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) return } c.JSON(http.StatusOK, gin.H{ + "status": "ok", "latest": latest, }) return @@ -52,6 +60,7 @@ func Init(db sql.Database, router *gin.Engine) { router.GET("/stats/cursor", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{ + "status": "ok", "current_cursor": cursor.Current, }) return From 30fa597fa788494fd66c98604326478fec7a457f Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Sun, 17 Jun 2018 20:52:22 +0200 Subject: [PATCH 08/18] Fix tests for JSON responses, add tests for invalid requests --- controllers/effects/effects_test.go | 40 ++++++++++++++++----- controllers/stats/stats_test.go | 55 ++++++++++++++++++++++++----- 2 files changed, 77 insertions(+), 18 deletions(-) diff --git a/controllers/effects/effects_test.go b/controllers/effects/effects_test.go index 16529e5..0414584 100644 --- a/controllers/effects/effects_test.go +++ b/controllers/effects/effects_test.go @@ -75,6 +75,15 @@ func TestEffects(t *testing.T) { http.StatusOK, test.Effects[3:5], }, + + // Invalid Filter{} + { + "GET", + "/effects?from=xxx", + "", + http.StatusBadRequest, + nil, + }, } router := gin.Default() @@ -91,19 +100,32 @@ func TestEffects(t *testing.T) { t.Errorf("Expected code %v, got %v, for %+v", test.statusCode, resp.Code, test) } - effects := make(map[string][]effect.Effect) - err := json.Unmarshal([]byte(resp.Body.String()), &effects) - if err != nil { - t.Error(err) + type resJson struct { + Status string + Effects []effect.Effect } - _, ok := effects["effects"] - if !ok { - t.Error(`Expected element "effects" in JSON response`) + if test.statusCode == http.StatusOK { + if !strings.Contains(resp.Body.String(), `"status":"ok"`) { + t.Errorf("Body did not contain ok status message: %s", resp.Body.String()) + } + } else { + if !strings.Contains(resp.Body.String(), `"status":"error"`) { + t.Errorf("Body did not contain error status message: %s", resp.Body.String()) + } + + // Skip to next test + continue + } + + res := resJson{} + err := json.Unmarshal([]byte(resp.Body.String()), &res) + if err != nil { + t.Error(err) } - if len(effects["effects"]) != len(test.expectedStats) { - t.Errorf("Expected %d JSON elements, got %d", len(test.expectedStats), len(effects["effects"])) + if len(res.Effects) != len(test.expectedStats) { + t.Errorf("Expected %d JSON elements, got %d", len(test.expectedStats), len(res.Effects)) } for _, e := range test.expectedStats { diff --git a/controllers/stats/stats_test.go b/controllers/stats/stats_test.go index b86831a..cfbbaa7 100644 --- a/controllers/stats/stats_test.go +++ b/controllers/stats/stats_test.go @@ -86,6 +86,15 @@ func TestStats(t *testing.T) { http.StatusOK, test.Effects[3:7], }, + + // Invalid Filter{} + { + "GET", + "/stats?from=xxx", + "", + http.StatusBadRequest, + nil, + }, } router := gin.Default() @@ -102,19 +111,32 @@ func TestStats(t *testing.T) { t.Errorf("Expected code %v, got %v, for %+v", tt.statusCode, resp.Code, tt) } - stats := make(map[string][]assetStat.AssetStat) - err := json.Unmarshal([]byte(resp.Body.String()), &stats) - if err != nil { - t.Error(err) + type resJson struct { + Status string + Stats []assetStat.AssetStat } - _, ok := stats["stats"] - if !ok { - t.Error(`Expected element "stats" in JSON response`) + if tt.statusCode == http.StatusOK { + if !strings.Contains(resp.Body.String(), `"status":"ok"`) { + t.Errorf("Body did not contain ok status message: %s", resp.Body.String()) + } + } else { + if !strings.Contains(resp.Body.String(), `"status":"error"`) { + t.Errorf("Body did not contain error status message: %s", resp.Body.String()) + } + + // Skip to next test + continue } - if len(stats["stats"]) != len(tt.expectedStats) { - t.Errorf("Expected %d JSON elements, got %d", len(tt.expectedStats), len(stats["stats"])) + res := resJson{} + err := json.Unmarshal([]byte(resp.Body.String()), &res) + if err != nil { + t.Error(err) + } + + if len(res.Stats) != len(tt.expectedStats) { + t.Errorf("Expected %d JSON elements, got %d", len(tt.expectedStats), len(res.Stats)) } for _, e := range tt.expectedStats { @@ -158,6 +180,7 @@ func TestLatestAndCursor(t *testing.T) { "", http.StatusOK, []string{ + `"status":"ok"`, fmt.Sprintf(`"paging_token":"%s"`, latestEffect.PagingToken), fmt.Sprintf(`"issued":"%s"`, bigint.ToString(latestEffect.Issued)), fmt.Sprintf(`"transferred":"%s"`, bigint.ToString(latestEffect.Transferred)), @@ -173,6 +196,7 @@ func TestLatestAndCursor(t *testing.T) { "", http.StatusOK, []string{ + `"status":"ok"`, fmt.Sprintf(`"current_cursor":"%s"`, cndy.GenesisCursor), }, }, @@ -196,6 +220,19 @@ func TestLatestAndCursor(t *testing.T) { t.Errorf("Expected code %v, got %v, for %+v", test.statusCode, resp.Code, test) } + if test.statusCode == http.StatusOK { + if !strings.Contains(resp.Body.String(), `"status":"ok"`) { + t.Errorf("Body did not contain ok status message: %s", resp.Body.String()) + } + } else { + if !strings.Contains(resp.Body.String(), `"status":"error"`) { + t.Errorf("Body did not contain error status message: %s", resp.Body.String()) + } + + // Skip to next test + continue + } + if len(test.bodyContains) > 0 { for _, s := range test.bodyContains { if !strings.Contains(resp.Body.String(), s) { From 7d12ecd7adfc4a456d1e6d54a2eb39cad0255bf2 Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Sun, 17 Jun 2018 20:57:04 +0200 Subject: [PATCH 09/18] Add status messages to documentation --- README.md | 4 + controllers/docs/docs.go | 182 ++++++++++++++++++++------------------- 2 files changed, 97 insertions(+), 89 deletions(-) diff --git a/README.md b/README.md index 9c8a375..9149026 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ GET https://api.cndy.store/stats/latest ```json { + "status": "ok", "latest": { "paging_token": "33825130903777281-1", "asset_type": "credit_alphanum4", @@ -80,6 +81,7 @@ If not set, `from` defaults to UNIX timestamp `0`, `to` to `now`. ```json { + "status": "ok", "stats": [ { "paging_token": "33864305300480001-1", @@ -115,6 +117,7 @@ GET https://api.cndy.store/stats/cursor ```json { + "status": "ok", "current_cursor": "33877250331906049-1" } ``` @@ -127,6 +130,7 @@ If not set, `from` defaults to UNIX timestamp `0`, `to` to `now`. ```json { + "status": "ok", "effects": [ { "id": "0033819672000335873-0000000001", diff --git a/controllers/docs/docs.go b/controllers/docs/docs.go index f02fdf8..6b8108e 100644 --- a/controllers/docs/docs.go +++ b/controllers/docs/docs.go @@ -16,107 +16,111 @@ func Init(router *gin.Engine) { // This data was generated from the corresponding README.md section using pandoc: // pandoc README.md -o docs.html const htmlData = ` -

API documentation

+

API endpoints and examples

Latest stats

GET https://api.cndy.store/stats/latest

+ "status": "ok", + "latest": { + "paging_token": "33825130903777281-1", + "asset_type": "credit_alphanum4", + "asset_code": "CNDY", + "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "payments": 4, + "accounts_with_trustline": 4, + "accounts_with_payments": 2, + "created_at": "2018-03-12T18:49:40Z", + "issued": "2000.0000000", + "transferred": "40.0000000" + } + }

Asset stats history

GET https://api.cndy.store/stats[?from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z]

If not set, from defaults to UNIX timestamp 0, to to now.

+ "status": "ok", + "stats": [ + { + "paging_token": "33864305300480001-1", + "asset_type": "credit_alphanum4", + "asset_code": "CNDY", + "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "payments": 6, + "accounts_with_trustline": 5, + "accounts_with_payments": 2, + "created_at": "2018-03-13T07:29:48Z", + "issued": "2000.0000000", + "transferred": "140.0000000" + }, + { + "paging_token": "33864305300480001-2", + "asset_type": "credit_alphanum4", + "asset_code": "CNDY", + "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "payments": 6, + "accounts_with_trustline": 5, + "accounts_with_payments": 2, + "created_at": "2018-03-13T07:29:48Z", + "issued": "3000.0000000", + "transferred": "140.0000000" + } + ] + }

Current Horizon cursor

GET https://api.cndy.store/stats/cursor

+ "status": "ok", + "current_cursor": "33877250331906049-1" + }

Effects

GET https://api.cndy.store/effects[?from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z]

If not set, from defaults to UNIX timestamp 0, to to now.

{
-	  "effects": [
-	    {
-	      "id": "0033819672000335873-0000000001",
-	      "operation": "https://horizon-testnet.stellar.org/operations/33819672000335873",
-	      "succeeds": "https://horizon-testnet.stellar.org/effects?order=desc&cursor=33819672000335873-1",
-	      "precedes": "https://horizon-testnet.stellar.org/effects?order=asc&cursor=33819672000335873-1",
-	      "paging_token": "33819672000335873-1",
-	      "account": "GDNH64DRUT4CY3UJLWQIB655PQ6OG34UGYB4NC5DC4TYWLNJIBCEYTTD",
-	      "type": "trustline_created",
-	      "type_i": 20,
-	      "starting_balance": "",
-	      "asset_type": "credit_alphanum4",
-	      "asset_code": "CNDY",
-	      "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
-	      "signer_public_key": "",
-	      "signer_weight": 0,
-	      "signer_key": "",
-	      "signer_type": "",
-	      "created_at": "2018-03-12T17:03:45Z",
-	      "amount": "0.0000000",
-	      "balance": "0.0000000",
-	      "balance_limit": "922337203685.4775807"
-	    },
-	    {
-	      "id": "0033820110087000065-0000000002",
-	      "operation": "https://horizon-testnet.stellar.org/operations/33820110087000065",
-	      "succeeds": "https://horizon-testnet.stellar.org/effects?order=desc&cursor=33820110087000065-2",
-	      "precedes": "https://horizon-testnet.stellar.org/effects?order=asc&cursor=33820110087000065-2",
-	      "paging_token": "33820110087000065-2",
-	      "account": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
-	      "type": "account_debited",
-	      "type_i": 3,
-	      "starting_balance": "",
-	      "asset_type": "credit_alphanum4",
-	      "asset_code": "CNDY",
-	      "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
-	      "signer_public_key": "",
-	      "signer_weight": 0,
-	      "signer_key": "",
-	      "signer_type": "",
-	      "created_at": "2018-03-12T17:12:15Z",
-	      "amount": "1000.0000000",
-	      "balance": "0.0000000",
-	      "balance_limit": "0.0000000"
-	    }
-	}
+ "status": "ok", + "effects": [ + { + "id": "0033819672000335873-0000000001", + "operation": "https://horizon-testnet.stellar.org/operations/33819672000335873", + "succeeds": "https://horizon-testnet.stellar.org/effects?order=desc&cursor=33819672000335873-1", + "precedes": "https://horizon-testnet.stellar.org/effects?order=asc&cursor=33819672000335873-1", + "paging_token": "33819672000335873-1", + "account": "GDNH64DRUT4CY3UJLWQIB655PQ6OG34UGYB4NC5DC4TYWLNJIBCEYTTD", + "type": "trustline_created", + "type_i": 20, + "starting_balance": "", + "asset_type": "credit_alphanum4", + "asset_code": "CNDY", + "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "signer_public_key": "", + "signer_weight": 0, + "signer_key": "", + "signer_type": "", + "created_at": "2018-03-12T17:03:45Z", + "amount": "0.0000000", + "balance": "0.0000000", + "balance_limit": "922337203685.4775807" + }, + { + "id": "0033820110087000065-0000000002", + "operation": "https://horizon-testnet.stellar.org/operations/33820110087000065", + "succeeds": "https://horizon-testnet.stellar.org/effects?order=desc&cursor=33820110087000065-2", + "precedes": "https://horizon-testnet.stellar.org/effects?order=asc&cursor=33820110087000065-2", + "paging_token": "33820110087000065-2", + "account": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "type": "account_debited", + "type_i": 3, + "starting_balance": "", + "asset_type": "credit_alphanum4", + "asset_code": "CNDY", + "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "signer_public_key": "", + "signer_weight": 0, + "signer_key": "", + "signer_type": "", + "created_at": "2018-03-12T17:12:15Z", + "amount": "1000.0000000", + "balance": "0.0000000", + "balance_limit": "0.0000000" + } + } ` From 6025cd8e37e2f6a809e8fb3d9f2a2e1031a04967 Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Sun, 17 Jun 2018 21:35:57 +0200 Subject: [PATCH 10/18] Fix bug where ID field was not ignored in response --- controllers/stats/stats_test.go | 5 +++++ models/asset_stat/asset_stat.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/controllers/stats/stats_test.go b/controllers/stats/stats_test.go index cfbbaa7..ea9a286 100644 --- a/controllers/stats/stats_test.go +++ b/controllers/stats/stats_test.go @@ -240,5 +240,10 @@ func TestLatestAndCursor(t *testing.T) { } } } + + // Check whether JSON ID is hidden (regression test) + if strings.Contains(resp.Body.String(), `"id":`) || strings.Contains(resp.Body.String(), `"Id":`) { + t.Errorf("Body did contain JSON ID (should be excluded) in '%s'", resp.Body.String()) + } } } diff --git a/models/asset_stat/asset_stat.go b/models/asset_stat/asset_stat.go index 75c7fa8..f819f13 100644 --- a/models/asset_stat/asset_stat.go +++ b/models/asset_stat/asset_stat.go @@ -9,7 +9,7 @@ import ( ) type AssetStat struct { - Id *uint32 `db:"id", json:"-"` + Id *uint32 `db:"id" json:"-"` PagingToken *string `db:"paging_token" json:"paging_token,omitempty"` AssetType *string `db:"asset_type" json:"asset_type,omitempty"` AssetCode *string `db:"asset_code" json:"asset_code,omitempty"` From 3858b9e49a0331d6359604b77d78fbee455ee67f Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Sun, 17 Jun 2018 23:09:23 +0200 Subject: [PATCH 11/18] Handle queries with no returned rows correctly, remove duplicate logs --- models/asset_stat/asset_stat.go | 13 ++++++++++--- models/effect/effect.go | 6 +++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/models/asset_stat/asset_stat.go b/models/asset_stat/asset_stat.go index f819f13..395102f 100644 --- a/models/asset_stat/asset_stat.go +++ b/models/asset_stat/asset_stat.go @@ -4,7 +4,6 @@ import ( "github.com/cndy-store/analytics/utils/bigint" "github.com/cndy-store/analytics/utils/sql" "github.com/stellar/go/clients/horizon" - "log" "time" ) @@ -68,7 +67,11 @@ func Get(db sql.Database, filter Filter) (stats []AssetStat, err error) { err = db.Select(&stats, `SELECT * FROM asset_stats WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY id`, filter.From, filter.To) if err == sql.ErrNoRows { - log.Printf("[ERROR] asset_stat.Get(): %s", err) + err = nil + return + } + if err != nil { + return } // Convert int64 fields to strings @@ -82,7 +85,11 @@ func Get(db sql.Database, filter Filter) (stats []AssetStat, err error) { func Latest(db sql.Database) (stats AssetStat, err error) { err = db.Get(&stats, `SELECT * FROM asset_stats ORDER BY id DESC LIMIT 1`) if err == sql.ErrNoRows { - log.Printf("[ERROR] asset_stat.Latest(): %s", err) + err = nil + return + } + if err != nil { + return } // Convert int64 fields to strings diff --git a/models/effect/effect.go b/models/effect/effect.go index 9ec8c96..f1a5092 100644 --- a/models/effect/effect.go +++ b/models/effect/effect.go @@ -146,7 +146,11 @@ func Get(db sql.Database, filter Filter) (effects []Effect, err error) { err = db.Select(&effects, `SELECT * FROM effects WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY created_at`, filter.From, filter.To) if err == sql.ErrNoRows { - log.Printf("[ERROR] effect.Get(): %s", err) + err = nil + return + } + if err != nil { + return } // Convert int64 fields to strings From edbe3435ec80a5dda0afab955116496b166f0350 Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Sun, 17 Jun 2018 23:24:37 +0200 Subject: [PATCH 12/18] Do not use ambigious testing name --- controllers/effects/effects_test.go | 18 +++++++++--------- controllers/stats/stats_test.go | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/controllers/effects/effects_test.go b/controllers/effects/effects_test.go index 0414584..ad3c755 100644 --- a/controllers/effects/effects_test.go +++ b/controllers/effects/effects_test.go @@ -89,15 +89,15 @@ func TestEffects(t *testing.T) { router := gin.Default() Init(tx, router) - for _, test := range tests { - body := bytes.NewBufferString(test.body) - req, _ := http.NewRequest(test.method, test.url, body) + for _, tt := range tests { + body := bytes.NewBufferString(tt.body) + req, _ := http.NewRequest(tt.method, tt.url, body) resp := httptest.NewRecorder() router.ServeHTTP(resp, req) - if resp.Code != test.statusCode { - t.Errorf("Expected code %v, got %v, for %+v", test.statusCode, resp.Code, test) + if resp.Code != tt.statusCode { + t.Errorf("Expected code %v, got %v, for %+v", tt.statusCode, resp.Code, tt) } type resJson struct { @@ -105,7 +105,7 @@ func TestEffects(t *testing.T) { Effects []effect.Effect } - if test.statusCode == http.StatusOK { + if tt.statusCode == http.StatusOK { if !strings.Contains(resp.Body.String(), `"status":"ok"`) { t.Errorf("Body did not contain ok status message: %s", resp.Body.String()) } @@ -124,11 +124,11 @@ func TestEffects(t *testing.T) { t.Error(err) } - if len(res.Effects) != len(test.expectedStats) { - t.Errorf("Expected %d JSON elements, got %d", len(test.expectedStats), len(res.Effects)) + if len(res.Effects) != len(tt.expectedStats) { + t.Errorf("Expected %d JSON elements, got %d", len(tt.expectedStats), len(res.Effects)) } - for _, e := range test.expectedStats { + for _, e := range tt.expectedStats { var s []string s = append(s, fmt.Sprintf(`"paging_token":"%s"`, e.PagingToken)) s = append(s, fmt.Sprintf(`"account":"%s"`, e.Account)) diff --git a/controllers/stats/stats_test.go b/controllers/stats/stats_test.go index ea9a286..2c428a0 100644 --- a/controllers/stats/stats_test.go +++ b/controllers/stats/stats_test.go @@ -209,18 +209,18 @@ func TestLatestAndCursor(t *testing.T) { } Init(tx, router) - for _, test := range tests { - body := bytes.NewBufferString(test.body) - req, _ := http.NewRequest(test.method, test.url, body) + for _, tt := range tests { + body := bytes.NewBufferString(tt.body) + req, _ := http.NewRequest(tt.method, tt.url, body) resp := httptest.NewRecorder() router.ServeHTTP(resp, req) - if resp.Code != test.statusCode { - t.Errorf("Expected code %v, got %v, for %+v", test.statusCode, resp.Code, test) + if resp.Code != tt.statusCode { + t.Errorf("Expected code %v, got %v, for %+v", tt.statusCode, resp.Code, tt) } - if test.statusCode == http.StatusOK { + if tt.statusCode == http.StatusOK { if !strings.Contains(resp.Body.String(), `"status":"ok"`) { t.Errorf("Body did not contain ok status message: %s", resp.Body.String()) } @@ -233,8 +233,8 @@ func TestLatestAndCursor(t *testing.T) { continue } - if len(test.bodyContains) > 0 { - for _, s := range test.bodyContains { + if len(tt.bodyContains) > 0 { + for _, s := range tt.bodyContains { if !strings.Contains(resp.Body.String(), s) { t.Errorf("Body did not contain '%s' in '%s'", s, resp.Body.String()) } From bd58a7a85e6774a7e8d03ee0f72643e2356911db Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Sun, 17 Jun 2018 15:17:22 +0200 Subject: [PATCH 13/18] Add basic asset migration, model and controller for multi-asset support --- controllers/assets/assets.go | 76 +++++++++++++++++++++++ db/migrations/0006_create_assets.down.sql | 1 + db/migrations/0006_create_assets.up.sql | 16 +++++ main.go | 36 +++++++---- models/asset/asset.go | 49 +++++++++++++++ 5 files changed, 166 insertions(+), 12 deletions(-) create mode 100644 controllers/assets/assets.go create mode 100644 db/migrations/0006_create_assets.down.sql create mode 100644 db/migrations/0006_create_assets.up.sql create mode 100644 models/asset/asset.go diff --git a/controllers/assets/assets.go b/controllers/assets/assets.go new file mode 100644 index 0000000..5f156ca --- /dev/null +++ b/controllers/assets/assets.go @@ -0,0 +1,76 @@ +package assets + +import ( + "fmt" + "github.com/cndy-store/analytics/models/asset" + "github.com/cndy-store/analytics/utils/sql" + "github.com/gin-gonic/gin" + "log" + "net/http" +) + +func Init(db sql.Database, router *gin.Engine) { + router.POST("/assets", func(c *gin.Context) { + // Read JSON body and parse it into asset struct + body := asset.Asset{} + err := c.BindJSON(&body) + if err != nil { + jsonErrorMsg := fmt.Sprintf("Invalid JSON body: %s", err) + c.JSON(http.StatusBadRequest, gin.H{ + "status": "error", + "message": jsonErrorMsg, + }) + return + } + + exists, err := asset.Exists(db, body) + if err != nil { + log.Printf("[ERROR] POST /assets: %s", err) + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) + return + } + if exists { + c.JSON(http.StatusConflict, gin.H{ + "status": "error", + "message": "Asset already exists", + }) + return + } + + newAsset, err := asset.New(db, body) + if err != nil { + log.Printf("[ERROR] POST /assets: %s", err) + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "status": "ok", + "asset": newAsset, + }) + }) + + router.GET("/assets", func(c *gin.Context) { + assets, err := asset.Get(db) + if err != nil { + log.Printf("[ERROR] Couldn't get assets from database: %s", err) + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "status": "ok", + "assets": assets, + }) + return + }) +} diff --git a/db/migrations/0006_create_assets.down.sql b/db/migrations/0006_create_assets.down.sql new file mode 100644 index 0000000..a2988f5 --- /dev/null +++ b/db/migrations/0006_create_assets.down.sql @@ -0,0 +1 @@ +DROP TABLE assets; diff --git a/db/migrations/0006_create_assets.up.sql b/db/migrations/0006_create_assets.up.sql new file mode 100644 index 0000000..03ba8c2 --- /dev/null +++ b/db/migrations/0006_create_assets.up.sql @@ -0,0 +1,16 @@ +CREATE TABLE assets ( + -- Code and issuer combination needs to be unique, can serve as primary key + code character varying(12) not null, + issuer character varying(56) not null, + primary key (code, issuer), + + type character varying(64) default 'credit_alphanum4', + created_at timestamp without time zone default current_timestamp +); + +-- Register CNDY +INSERT INTO assets(type, code, issuer) VALUES( + 'credit_alphanum4', + 'CNDY', + 'GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX' +); diff --git a/main.go b/main.go index 7583fdc..7008fac 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,13 @@ package main import ( + "github.com/cndy-store/analytics/controllers/assets" "github.com/cndy-store/analytics/controllers/docs" "github.com/cndy-store/analytics/controllers/effects" "github.com/cndy-store/analytics/controllers/stats" + "github.com/cndy-store/analytics/models/asset" "github.com/cndy-store/analytics/models/cursor" "github.com/cndy-store/analytics/models/effect" - "github.com/cndy-store/analytics/utils/cndy" "github.com/cndy-store/analytics/utils/sql" "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" @@ -63,18 +64,28 @@ func main() { os.Exit(1) } + // Load registered assets into asset.Registered + asset.UpdateRegistered(db) + if err != nil { + log.Printf("[ERROR] Couldn't get registered assets from database: %s", err) + os.Exit(1) + } + for { client.StreamEffects(ctx, &cursor.Current, func(e horizon.Effect) { - if e.Asset.Code == cndy.AssetCode && e.Asset.Issuer == cndy.AssetIssuer { - err = effect.New(db, e) - if err != nil { - log.Printf("[ERROR] Couldn't save effect to database: %s", err) - } - - // Make sure to also safe the current cursor, so database is consistent - err = cursor.Save(db) - if err != nil { - log.Printf("[ERROR] Couldn't save cursor to database: %s", err) + // Check whether this asset was registered + for _, registeredAsset := range asset.Registered { + if e.Asset.Code == *registeredAsset.Code && e.Asset.Issuer == *registeredAsset.Issuer { + err = effect.New(db, e) + if err != nil { + log.Printf("[ERROR] Couldn't save effect to database: %s", err) + } + + // Make sure to also safe the current cursor, so database is consistent + err = cursor.Save(db) + if err != nil { + log.Printf("[ERROR] Couldn't save cursor to database: %s", err) + } } } @@ -87,8 +98,9 @@ func api(db *sqlx.DB) { router := gin.Default() router.Use(cors.Default()) // Allow all origins - stats.Init(db, router) + assets.Init(db, router) effects.Init(db, router) + stats.Init(db, router) docs.Init(router) router.Run(":3144") diff --git a/models/asset/asset.go b/models/asset/asset.go new file mode 100644 index 0000000..a3f62bf --- /dev/null +++ b/models/asset/asset.go @@ -0,0 +1,49 @@ +package asset + +import ( + "github.com/cndy-store/analytics/utils/sql" + "time" +) + +type Asset struct { + Type *string `db:"type" json:"type,omitempty"` + Code *string `db:"code" json:"code,omitempty" binding="required"` + Issuer *string `db:"issuer" json:"issuer,omitempty binding="required"` + CreatedAt *time.Time `db:"created_at" json:"created_at,omitempty"` +} + +// Variable to hold all registered assets in memory +var Registered []Asset + +func New(db sql.Database, asset Asset) (ret Asset, err error) { + err = db.Get(&ret, `INSERT INTO assets(type, code, issuer) VALUES($1, $2, $3) RETURNING *`, + asset.Type, asset.Code, asset.Issuer) + if err != nil { + return + } + + err = UpdateRegistered(db) + return +} + +func Exists(db sql.Database, asset Asset) (yes bool, err error) { + return sql.Exists(db, `SELECT 1 FROM assets WHERE code=$1 AND issuer=$2`, asset.Code, asset.Issuer) +} + +func Get(db sql.Database) (assets []Asset, err error) { + err = db.Select(&assets, `SELECT * FROM assets`) + if err == sql.ErrNoRows { + err = nil + return + } + if err != nil { + return + } + + return +} + +func UpdateRegistered(db sql.Database) (err error) { + Registered, err = Get(db) + return +} From 5d278c7d9d3312a973c464ad97e20b05d7c2ac62 Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Wed, 4 Jul 2018 21:34:16 +0200 Subject: [PATCH 14/18] Extend filter to include asset_code and asset_issuer --- controllers/effects/effects.go | 4 +- controllers/effects/effects_test.go | 20 ++++++--- controllers/stats/stats.go | 16 ++++++-- controllers/stats/stats_test.go | 30 +++++++++++--- models/asset_stat/asset_stat.go | 30 ++++---------- models/asset_stat/asset_stat_test.go | 11 ++--- models/effect/effect.go | 25 ++---------- models/effect/effect_test.go | 16 ++++---- utils/filter/filter.go | 61 ++++++++++++++++++++++++++-- 9 files changed, 138 insertions(+), 75 deletions(-) diff --git a/controllers/effects/effects.go b/controllers/effects/effects.go index c9c7c01..41cc833 100644 --- a/controllers/effects/effects.go +++ b/controllers/effects/effects.go @@ -12,7 +12,7 @@ import ( func Init(db sql.Database, router *gin.Engine) { // GET /effects[?from=XXX&to=XXX] router.GET("/effects", func(c *gin.Context) { - from, to, err := filter.Parse(c) + args, err := filter.Parse(c) if err != nil { log.Printf("[ERROR] Couldn't parse URL parameters: %s", err) c.JSON(http.StatusBadRequest, gin.H{ @@ -22,7 +22,7 @@ func Init(db sql.Database, router *gin.Engine) { return } - effects, err := effect.Get(db, effect.Filter{From: from, To: to}) + effects, err := effect.Get(db, args) if err != nil { log.Printf("[ERROR] Couldn't get effect from database: %s", err) c.JSON(http.StatusInternalServerError, gin.H{ diff --git a/controllers/effects/effects_test.go b/controllers/effects/effects_test.go index ad3c755..3f3eedb 100644 --- a/controllers/effects/effects_test.go +++ b/controllers/effects/effects_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/cndy-store/analytics/models/effect" + "github.com/cndy-store/analytics/utils/cndy" "github.com/cndy-store/analytics/utils/sql" "github.com/cndy-store/analytics/utils/test" "github.com/gin-gonic/gin" @@ -43,7 +44,7 @@ func TestEffects(t *testing.T) { var tests = []HttpTest{ { "GET", - "/effects", + fmt.Sprintf("/effects?asset_code=%s&asset_issuer=%s", cndy.AssetCode, cndy.AssetIssuer), "", http.StatusOK, test.Effects, @@ -52,7 +53,7 @@ func TestEffects(t *testing.T) { // Filter{From} { "GET", - fmt.Sprintf("/effects?from=%s", test.Effects[5].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/effects?asset_code=%s&asset_issuer=%s&from=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[5].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[5:], @@ -61,7 +62,7 @@ func TestEffects(t *testing.T) { // Filter{To} { "GET", - fmt.Sprintf("/effects?to=%s", test.Effects[2].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/effects?asset_code=%s&asset_issuer=%s&to=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[2].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[:3], @@ -70,7 +71,7 @@ func TestEffects(t *testing.T) { // Filter{From, To} { "GET", - fmt.Sprintf("/effects?from=%s&to=%s", test.Effects[3].CreatedAt.Format(time.RFC3339), test.Effects[4].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/effects?asset_code=%s&asset_issuer=%s&from=%s&to=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[3].CreatedAt.Format(time.RFC3339), test.Effects[4].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[3:5], @@ -79,7 +80,16 @@ func TestEffects(t *testing.T) { // Invalid Filter{} { "GET", - "/effects?from=xxx", + fmt.Sprintf("/effects?asset_code=%s&asset_issuer=%s&from=xxx", cndy.AssetCode, cndy.AssetIssuer), + "", + http.StatusBadRequest, + nil, + }, + + // Missing asset_code and asset_issuer + { + "GET", + "/effects", "", http.StatusBadRequest, nil, diff --git a/controllers/stats/stats.go b/controllers/stats/stats.go index c8d2b16..49b67a5 100644 --- a/controllers/stats/stats.go +++ b/controllers/stats/stats.go @@ -13,7 +13,7 @@ import ( func Init(db sql.Database, router *gin.Engine) { // GET /stats[?from=XXX&to=XXX] router.GET("/stats", func(c *gin.Context) { - from, to, err := filter.Parse(c) + args, err := filter.Parse(c) if err != nil { log.Printf("[ERROR] Couldn't parse URL parameters: %s", err) c.JSON(http.StatusBadRequest, gin.H{ @@ -23,7 +23,7 @@ func Init(db sql.Database, router *gin.Engine) { return } - assetStats, err := assetStat.Get(db, assetStat.Filter{From: from, To: to}) + assetStats, err := assetStat.Get(db, args) if err != nil { log.Printf("[ERROR] Couldn't get asset stats from database: %s", err) c.JSON(http.StatusInternalServerError, gin.H{ @@ -41,7 +41,17 @@ func Init(db sql.Database, router *gin.Engine) { }) router.GET("/stats/latest", func(c *gin.Context) { - latest, err := assetStat.Latest(db) + args, err := filter.Parse(c) + if err != nil { + log.Printf("[ERROR] Couldn't parse URL parameters: %s", err) + c.JSON(http.StatusBadRequest, gin.H{ + "status": "error", + "message": err.Error(), + }) + return + } + + latest, err := assetStat.Latest(db, args) if err != nil { log.Printf("[ERROR] Couldn't get asset stats from database: %s", err) c.JSON(http.StatusInternalServerError, gin.H{ diff --git a/controllers/stats/stats_test.go b/controllers/stats/stats_test.go index 2c428a0..7b816ae 100644 --- a/controllers/stats/stats_test.go +++ b/controllers/stats/stats_test.go @@ -54,7 +54,7 @@ func TestStats(t *testing.T) { var tests = []HttpTestWithEffects{ { "GET", - "/stats", + fmt.Sprintf("/stats?asset_code=%s&asset_issuer=%s", cndy.AssetCode, cndy.AssetIssuer), "", http.StatusOK, test.Effects, @@ -63,7 +63,7 @@ func TestStats(t *testing.T) { // Filter{From} { "GET", - fmt.Sprintf("/stats?from=%s", test.Effects[4].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/stats?asset_code=%s&asset_issuer=%s&from=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[4].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[4:], @@ -72,7 +72,7 @@ func TestStats(t *testing.T) { // Filter{To} { "GET", - fmt.Sprintf("/stats?to=%s", test.Effects[2].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/stats?asset_code=%s&asset_issuer=%s&to=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[2].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[:3], @@ -81,7 +81,7 @@ func TestStats(t *testing.T) { // Filter{From, To} { "GET", - fmt.Sprintf("/stats?from=%s&to=%s", test.Effects[3].CreatedAt.Format(time.RFC3339), test.Effects[6].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/stats?asset_code=%s&asset_issuer=%s&from=%s&to=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[3].CreatedAt.Format(time.RFC3339), test.Effects[6].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[3:7], @@ -90,7 +90,16 @@ func TestStats(t *testing.T) { // Invalid Filter{} { "GET", - "/stats?from=xxx", + fmt.Sprintf("/stats?asset_code=%s&asset_issuer=%s&from=xxx", cndy.AssetCode, cndy.AssetIssuer), + "", + http.StatusBadRequest, + nil, + }, + + // Missing asset_code and asset_issuer + { + "GET", + "/stats", "", http.StatusBadRequest, nil, @@ -176,7 +185,7 @@ func TestLatestAndCursor(t *testing.T) { var tests = []HttpTest{ { "GET", - "/stats/latest", + fmt.Sprintf("/stats/latest?asset_code=%s&asset_issuer=%s", cndy.AssetCode, cndy.AssetIssuer), "", http.StatusOK, []string{ @@ -190,6 +199,15 @@ func TestLatestAndCursor(t *testing.T) { }, }, + // Missing asset_code and asset_issuer + { + "GET", + "/stats/latest", + "", + http.StatusBadRequest, + nil, + }, + { "GET", "/stats/cursor", diff --git a/models/asset_stat/asset_stat.go b/models/asset_stat/asset_stat.go index 395102f..2df97a5 100644 --- a/models/asset_stat/asset_stat.go +++ b/models/asset_stat/asset_stat.go @@ -2,6 +2,7 @@ package assetStat import ( "github.com/cndy-store/analytics/utils/bigint" + "github.com/cndy-store/analytics/utils/filter" "github.com/cndy-store/analytics/utils/sql" "github.com/stellar/go/clients/horizon" "time" @@ -45,27 +46,10 @@ func New(db sql.Database, effect horizon.Effect, timestamp time.Time) (err error return } -type Filter struct { - From *time.Time - To *time.Time -} - -func (f *Filter) Defaults() { - if f.From == nil { - t := time.Unix(0, 0) - f.From = &t - } - - if f.To == nil { - t := time.Now() - f.To = &t - } -} - -func Get(db sql.Database, filter Filter) (stats []AssetStat, err error) { +func Get(db sql.Database, filter filter.Filter) (stats []AssetStat, err error) { filter.Defaults() - err = db.Select(&stats, `SELECT * FROM asset_stats WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY id`, - filter.From, filter.To) + err = db.Select(&stats, `SELECT * FROM asset_stats WHERE asset_code=$1 AND asset_issuer=$2 AND created_at BETWEEN $3::timestamp AND $4::timestamp ORDER BY id`, + filter.AssetCode, filter.AssetIssuer, filter.From, filter.To) if err == sql.ErrNoRows { err = nil return @@ -82,8 +66,10 @@ func Get(db sql.Database, filter Filter) (stats []AssetStat, err error) { return } -func Latest(db sql.Database) (stats AssetStat, err error) { - err = db.Get(&stats, `SELECT * FROM asset_stats ORDER BY id DESC LIMIT 1`) +func Latest(db sql.Database, filter filter.Filter) (stats AssetStat, err error) { + filter.Defaults() + err = db.Get(&stats, `SELECT * FROM asset_stats WHERE asset_code=$1 AND asset_issuer=$2 ORDER BY id DESC LIMIT 1`, + filter.AssetCode, filter.AssetIssuer) if err == sql.ErrNoRows { err = nil return diff --git a/models/asset_stat/asset_stat_test.go b/models/asset_stat/asset_stat_test.go index e087071..51434ae 100644 --- a/models/asset_stat/asset_stat_test.go +++ b/models/asset_stat/asset_stat_test.go @@ -1,6 +1,7 @@ package assetStat import ( + "github.com/cndy-store/analytics/utils/filter" "github.com/cndy-store/analytics/utils/sql" "github.com/cndy-store/analytics/utils/test" "testing" @@ -24,7 +25,7 @@ func TestGet(t *testing.T) { } // Filter{} - assetStats, err := Get(tx, Filter{}) + assetStats, err := Get(tx, filter.NewCNDYFilter(nil, nil)) if err != nil { t.Errorf("assetStat.Get(): %s", err) } @@ -33,7 +34,7 @@ func TestGet(t *testing.T) { } // Filter{From} - assetStats, err = Get(tx, Filter{From: &test.Effects[2].CreatedAt}) + assetStats, err = Get(tx, filter.NewCNDYFilter(&test.Effects[2].CreatedAt, nil)) if err != nil { t.Errorf("assetStat.Get(): %s", err) } @@ -64,7 +65,7 @@ func TestGet(t *testing.T) { } // Filter{To} - assetStats, err = Get(tx, Filter{To: &test.Effects[1].CreatedAt}) + assetStats, err = Get(tx, filter.NewCNDYFilter(nil, &test.Effects[1].CreatedAt)) if err != nil { t.Errorf("assetStat.Get(): %s", err) } @@ -95,7 +96,7 @@ func TestGet(t *testing.T) { } // Filter{From, To} - assetStats, err = Get(tx, Filter{From: &test.Effects[1].CreatedAt, To: &test.Effects[2].CreatedAt}) + assetStats, err = Get(tx, filter.NewCNDYFilter(&test.Effects[1].CreatedAt, &test.Effects[2].CreatedAt)) if err != nil { t.Errorf("assetStat.Get(): %s", err) } @@ -144,7 +145,7 @@ func TestLatest(t *testing.T) { } // Filter{} - assetStats, err := Latest(tx) + assetStats, err := Latest(tx, filter.NewCNDYFilter(nil, nil)) if err != nil { t.Errorf("assetStat.Latest(): %s", err) } diff --git a/models/effect/effect.go b/models/effect/effect.go index f1a5092..852e903 100644 --- a/models/effect/effect.go +++ b/models/effect/effect.go @@ -4,6 +4,7 @@ import ( "encoding/json" "github.com/cndy-store/analytics/models/asset_stat" "github.com/cndy-store/analytics/utils/bigint" + "github.com/cndy-store/analytics/utils/filter" "github.com/cndy-store/analytics/utils/sql" "github.com/stellar/go/clients/horizon" "log" @@ -123,28 +124,10 @@ func New(db sql.Database, effect horizon.Effect) (err error) { return } -type Filter struct { - Type string - From *time.Time - To *time.Time -} - -func (f *Filter) Defaults() { - if f.From == nil { - t := time.Unix(0, 0) - f.From = &t - } - - if f.To == nil { - t := time.Now() - f.To = &t - } -} - -func Get(db sql.Database, filter Filter) (effects []Effect, err error) { +func Get(db sql.Database, filter filter.Filter) (effects []Effect, err error) { filter.Defaults() - err = db.Select(&effects, `SELECT * FROM effects WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY created_at`, - filter.From, filter.To) + err = db.Select(&effects, `SELECT * FROM effects WHERE asset_code=$1 AND asset_issuer=$2 AND created_at BETWEEN $3::timestamp AND $4::timestamp ORDER BY created_at`, + filter.AssetCode, filter.AssetIssuer, filter.From, filter.To) if err == sql.ErrNoRows { err = nil return diff --git a/models/effect/effect_test.go b/models/effect/effect_test.go index 4129dce..01d6933 100644 --- a/models/effect/effect_test.go +++ b/models/effect/effect_test.go @@ -4,6 +4,7 @@ import ( "github.com/cndy-store/analytics/models/asset_stat" "github.com/cndy-store/analytics/utils/bigint" "github.com/cndy-store/analytics/utils/cndy" + "github.com/cndy-store/analytics/utils/filter" "github.com/cndy-store/analytics/utils/sql" "github.com/cndy-store/analytics/utils/test" "github.com/stellar/go/clients/horizon" @@ -70,7 +71,7 @@ func TestNew(t *testing.T) { t.Error(err) } - effects, err := Get(tx, Filter{}) + effects, err := Get(tx, filter.NewCNDYFilter(nil, nil)) if err != nil { t.Error(err) } @@ -130,7 +131,7 @@ func TestNew(t *testing.T) { } // Check whether asset_stat data was updated - a, err := assetStat.Latest(tx) + a, err := assetStat.Latest(tx, filter.NewCNDYFilter(nil, nil)) if err != nil { t.Error(err) } @@ -175,8 +176,8 @@ func TestGet(t *testing.T) { t.Error(err) } - // Filter{} - effects, err := Get(tx, Filter{}) + // NewCNDYFilter(nil, nil) + effects, err := Get(tx, filter.NewCNDYFilter(nil, nil)) if err != nil { t.Errorf("effect.Get(): %s", err) } @@ -185,7 +186,7 @@ func TestGet(t *testing.T) { } // Filter{From} - effects, err = Get(tx, Filter{From: &test.Effects[5].CreatedAt}) + effects, err = Get(tx, filter.NewCNDYFilter(&test.Effects[5].CreatedAt, nil)) if err != nil { t.Errorf("effect.Get(): %s", err) } @@ -201,7 +202,7 @@ func TestGet(t *testing.T) { } // Filter{To} - effects, err = Get(tx, Filter{To: &test.Effects[2].CreatedAt}) + effects, err = Get(tx, filter.NewCNDYFilter(nil, &test.Effects[2].CreatedAt)) if err != nil { t.Errorf("effect.Get(): %s", err) } @@ -217,7 +218,7 @@ func TestGet(t *testing.T) { } // Filter{From, To} - effects, err = Get(tx, Filter{From: &test.Effects[3].CreatedAt, To: &test.Effects[4].CreatedAt}) + effects, err = Get(tx, filter.NewCNDYFilter(&test.Effects[3].CreatedAt, &test.Effects[4].CreatedAt)) if err != nil { t.Errorf("effect.Get(): %s", err) } @@ -226,7 +227,6 @@ func TestGet(t *testing.T) { } for i, e := range test.Effects[3:5] { - if e.PagingToken != *effects[i].PagingToken { t.Errorf("Expected paging_token to be %s got: %s", e.PagingToken, *effects[i].PagingToken) } diff --git a/utils/filter/filter.go b/utils/filter/filter.go index a6737e9..ac945b2 100644 --- a/utils/filter/filter.go +++ b/utils/filter/filter.go @@ -2,18 +2,26 @@ package filter import ( "errors" + "github.com/cndy-store/analytics/utils/cndy" "github.com/gin-gonic/gin" "time" ) -func Parse(c *gin.Context) (from *time.Time, to *time.Time, err error) { +type Filter struct { + From *time.Time + To *time.Time + AssetCode string + AssetIssuer string +} + +func Parse(c *gin.Context) (filter Filter, err error) { if query := c.Query("from"); query != "" { t, e := time.Parse(time.RFC3339, query) if e != nil { err = errors.New("Invalid date in 'from' parameter.") return } - from = &t + filter.From = &t } if query := c.Query("to"); query != "" { @@ -22,8 +30,55 @@ func Parse(c *gin.Context) (from *time.Time, to *time.Time, err error) { err = errors.New("Invalid date in 'to' parameter.") return } - to = &t + filter.To = &t + } + + if query := c.Query("asset_code"); query != "" { + filter.AssetCode = query + } else { + err = errors.New("Missing 'asset_code' parameter") + return + } + + if query := c.Query("asset_issuer"); query != "" { + filter.AssetIssuer = query + } else { + err = errors.New("Missing 'asset_issuer' parameter") + return } return } + +func (f *Filter) Defaults() { + if f.From == nil { + t := time.Unix(0, 0) + f.From = &t + } + + if f.To == nil { + t := time.Now() + f.To = &t + } +} + +// Returns a filter object with pre-filled AssetIssuer and AssetCode +// for the CNDY coin used in testing +func NewCNDYFilter(from *time.Time, to *time.Time) Filter { + if from == nil { + t := time.Unix(0, 0) + from = &t + } + + if to == nil { + t := time.Now() + to = &t + } + + return Filter{ + From: from, + To: to, + AssetCode: cndy.AssetCode, + AssetIssuer: cndy.AssetIssuer, + } +} From afc457d92b85b74a420385dacd2e770124072e65 Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Wed, 4 Jul 2018 21:36:38 +0200 Subject: [PATCH 15/18] Update documentation to include new filters --- README.md | 7 ++++--- controllers/docs/docs.go | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9149026..ed2647e 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ docker run -v $PWD:/host --entrypoint cp cndy-store/analytics analytics /host/cn ## Latest stats -GET https://api.cndy.store/stats/latest +GET https://api.cndy.store/stats/latest?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX ```json { @@ -75,7 +75,7 @@ GET https://api.cndy.store/stats/latest ## Asset stats history -GET https://api.cndy.store/stats[?from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z] +GET https://api.cndy.store/stats?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX[&from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z] If not set, `from` defaults to UNIX timestamp `0`, `to` to `now`. @@ -124,7 +124,8 @@ GET https://api.cndy.store/stats/cursor ## Effects -GET https://api.cndy.store/effects[?from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z] +GET https://api.cndy.store/effects?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX[&from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z] + If not set, `from` defaults to UNIX timestamp `0`, `to` to `now`. diff --git a/controllers/docs/docs.go b/controllers/docs/docs.go index 6b8108e..6b9d585 100644 --- a/controllers/docs/docs.go +++ b/controllers/docs/docs.go @@ -18,7 +18,7 @@ func Init(router *gin.Engine) { const htmlData = `

API endpoints and examples

Latest stats

-

GET https://api.cndy.store/stats/latest

+

GET https://api.cndy.store/stats/latest?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX

{
 	  "status": "ok",
 	  "latest": {
@@ -35,7 +35,7 @@ const htmlData = `
 	  }
 	}

Asset stats history

-

GET https://api.cndy.store/stats[?from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z]

+

GET https://api.cndy.store/stats?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX[&from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z]

If not set, from defaults to UNIX timestamp 0, to to now.

Effects

-

GET https://api.cndy.store/effects[?from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z]

+

GET https://api.cndy.store/effects?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX[&from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z]

If not set, from defaults to UNIX timestamp 0, to to now.

{
 	  "status": "ok",

From b3f656bb799aebbc0795baed8fc4ccbaded76dee Mon Sep 17 00:00:00 2001
From: Chris Aumann 
Date: Wed, 4 Jul 2018 22:25:22 +0200
Subject: [PATCH 16/18] Add documentation for asset controller

---
 README.md                | 53 ++++++++++++++++++++++++++++++++++++++++
 controllers/docs/docs.go | 35 ++++++++++++++++++++++++++
 2 files changed, 88 insertions(+)

diff --git a/README.md b/README.md
index ed2647e..e673913 100644
--- a/README.md
+++ b/README.md
@@ -179,3 +179,56 @@ If not set, `from` defaults to UNIX timestamp `0`, `to` to `now`.
     }
 }
 ```
+
+
+## Assets
+
+### Create a new asset
+
+POST https://api.cndy.store/assets
+
+Body
+
+```json
+{
+  "code": "CNDY",
+  "issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX"
+}
+```
+
+Response
+
+```json
+{
+  "status": "ok",
+  "asset": {
+    "code": "CNDY",
+    "issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
+    "created_at": "2018-07-04T19:16:47.02965Z"
+  }
+}
+```
+
+### Get all known assets
+
+GET https://api.cndy.store/assets
+
+
+```json
+{
+  "status": "ok",
+  "assets": [
+    {
+      "type": "credit_alphanum4",
+      "code": "CNDY",
+      "issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
+      "created_at": "2018-07-04T19:16:47.02965Z"
+    },
+    {
+      "code": "LOCALCOIN",
+      "issuer": "GCJKCXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+      "created_at": "2018-07-04T19:54:39.14328Z"
+    }
+  ]
+}
+```
diff --git a/controllers/docs/docs.go b/controllers/docs/docs.go
index 6b9d585..84ab695 100644
--- a/controllers/docs/docs.go
+++ b/controllers/docs/docs.go
@@ -123,4 +123,39 @@ const htmlData = `
 	      "balance_limit": "0.0000000"
 	    }
 	}
+

Assets

+

Create a new asset

+

POST https://api.cndy.store/assets

+

Body

+ +

Response

+ +

Get all known assets

+

GET https://api.cndy.store/assets

+ ` From f8b07df33939b252dce7c4964101185f3842146b Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Wed, 4 Jul 2018 22:37:13 +0200 Subject: [PATCH 17/18] Add integration tests for asset controller --- controllers/assets/assets_test.go | 136 ++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 controllers/assets/assets_test.go diff --git a/controllers/assets/assets_test.go b/controllers/assets/assets_test.go new file mode 100644 index 0000000..0682f30 --- /dev/null +++ b/controllers/assets/assets_test.go @@ -0,0 +1,136 @@ +package assets + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/cndy-store/analytics/utils/cndy" + "github.com/cndy-store/analytics/utils/sql" + "github.com/cndy-store/analytics/utils/test" + "github.com/gin-gonic/gin" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +type HttpTest struct { + method string + url string + body string + statusCode int + expectedBody []string +} + +func TestAssets(t *testing.T) { + db, err := sql.OpenAndMigrate("../..") + if err != nil { + t.Error(err) + } + + tx, err := db.Beginx() + if err != nil { + t.Error(err) + } + defer tx.Rollback() + + err = test.InsertTestData(tx) + if err != nil { + t.Error(err) + } + + var tests = []HttpTest{ + { + "POST", + "/assets", + `{"code": "TEST", "issuer": "GCJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}`, + http.StatusOK, + []string{ + `"status":"ok"`, + `"code":"TEST"`, + `"issuer":"GCJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"`, + }, + }, + + // Check whether duplicates are prevented + { + "POST", + "/assets", + `{"code": "TEST", "issuer": "GCJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}`, + http.StatusConflict, + []string{ + `"status":"error"`, + `"message":"Asset already exists"`, + }, + }, + + { + "POST", + "/assets", + `{"code": "invalid`, + http.StatusBadRequest, + []string{ + `"status":"error"`, + }, + }, + + // Check whether new asset as well as CNDY asset are present in database + { + "GET", + "/assets", + "", + http.StatusOK, + []string{ + `"status":"ok"`, + fmt.Sprintf(`"code":"%s"`, cndy.AssetCode), + fmt.Sprintf(`"issuer":"%s"`, cndy.AssetIssuer), + `"code":"TEST"`, + `"issuer":"GCJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"`, + }, + }, + } + + router := gin.Default() + Init(tx, router) + + for _, tt := range tests { + body := bytes.NewBufferString(tt.body) + req, _ := http.NewRequest(tt.method, tt.url, body) + resp := httptest.NewRecorder() + + router.ServeHTTP(resp, req) + + if resp.Code != tt.statusCode { + t.Errorf("Expected code %v, got %v, for %+v", tt.statusCode, resp.Code, tt) + } + + type resJson struct { + Status string + } + + if tt.statusCode == http.StatusOK { + if !strings.Contains(resp.Body.String(), `"status":"ok"`) { + t.Errorf("Body did not contain ok status message: %s", resp.Body.String()) + } + } else { + if !strings.Contains(resp.Body.String(), `"status":"error"`) { + t.Errorf("Body did not contain error status message: %s", resp.Body.String()) + } + + // Skip to next test + continue + } + + res := resJson{} + err := json.Unmarshal([]byte(resp.Body.String()), &res) + if err != nil { + t.Error(err) + } + + for _, contains := range tt.expectedBody { + if !strings.Contains(resp.Body.String(), contains) { + t.Errorf("Body did not contain '%s' in '%s'", contains, resp.Body.String()) + } + } + } +} From f22c26b8431e30bd0b37f7053287f4eb14625222 Mon Sep 17 00:00:00 2001 From: Chris Aumann Date: Mon, 11 Jun 2018 23:53:29 +0200 Subject: [PATCH 18/18] Use asset_stats() stored procedure instead of table --- .../0007_asset_stats_procedure.down.sql | 39 ++++++++++++++++ .../0007_asset_stats_procedure.up.sql | 44 +++++++++++++++++++ models/asset_stat/asset_stat.go | 33 +++++++------- models/effect/effect.go | 7 --- utils/test/test.go | 5 --- 5 files changed, 99 insertions(+), 29 deletions(-) create mode 100644 db/migrations/0007_asset_stats_procedure.down.sql create mode 100644 db/migrations/0007_asset_stats_procedure.up.sql diff --git a/db/migrations/0007_asset_stats_procedure.down.sql b/db/migrations/0007_asset_stats_procedure.down.sql new file mode 100644 index 0000000..5001801 --- /dev/null +++ b/db/migrations/0007_asset_stats_procedure.down.sql @@ -0,0 +1,39 @@ +-- Recreate asset_stats table (taken from migration 0005) +CREATE TABLE asset_stats ( + id serial PRIMARY KEY, + paging_token character varying(64), + asset_type character varying(64), + asset_code character varying(12), + asset_issuer character varying(56), + issued bigint, + transferred bigint, + accounts_with_trustline integer, + accounts_with_payments integer, + payments integer, + created_at timestamp without time zone +); + +CREATE OR REPLACE FUNCTION repopulate_asset_stats() + RETURNS VOID +AS +$$ +DECLARE + t_row record; +BEGIN + TRUNCATE asset_stats; + FOR t_row in SELECT paging_token, effect_id, asset_type, asset_code, asset_issuer, created_at FROM effects ORDER BY effect_id LOOP + INSERT INTO asset_stats(paging_token, asset_code, asset_issuer, asset_type, created_at, issued, transferred, accounts_with_trustline, accounts_with_payments, payments) + VALUES (t_row.paging_token, t_row.asset_code, t_row.asset_issuer, t_row.asset_type, t_row.created_at, + (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type = 'account_debited' AND account = t_row.asset_issuer AND effect_id <= t_row.effect_id), + (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type = 'account_debited' AND account != t_row.asset_issuer AND effect_id <= t_row.effect_id), + (SELECT COUNT(DISTINCT account) FROM effects WHERE account != t_row.asset_issuer AND type = 'trustline_created' AND effect_id <= t_row.effect_id), + (SELECT COUNT(DISTINCT account) FROM effects WHERE account != t_row.asset_issuer AND type = 'account_debited' AND effect_id <= t_row.effect_id), + (SELECT COUNT(*) FROM effects WHERE type = 'account_debited' AND account != t_row.asset_issuer AND effect_id <= t_row.effect_id) + ); + END LOOP; +END; +$$ +LANGUAGE plpgsql; + +-- Re-populate asset_stats table +SELECT repopulate_asset_stats(); diff --git a/db/migrations/0007_asset_stats_procedure.up.sql b/db/migrations/0007_asset_stats_procedure.up.sql new file mode 100644 index 0000000..c2a3cbe --- /dev/null +++ b/db/migrations/0007_asset_stats_procedure.up.sql @@ -0,0 +1,44 @@ +CREATE OR REPLACE FUNCTION asset_stats(asset_code_filter character varying(12), asset_issuer_filter character varying(56)) + RETURNS TABLE ( + -- TODO: ID to order required? + effect_id character varying(56), + paging_token character varying(64), + asset_type character varying(64), + asset_code character varying(12), + asset_issuer character varying(56), + issued bigint, + transferred bigint, + accounts_with_trustline integer, + accounts_with_payments integer, + payments integer, + created_at timestamp without time zone + ) +AS +$$ +#variable_conflict use_column +DECLARE + t_row record; +BEGIN + FOR t_row in SELECT effect_id, paging_token, asset_type, asset_code, asset_issuer, created_at FROM effects ORDER BY created_at LOOP + -- Next if asset_code and asset_issuer do not match + CONTINUE WHEN t_row.asset_code != asset_code_filter AND t_row.asset_issuer != asset_issuer_filter; + + effect_id := t_row.effect_id; + paging_token := t_row.paging_token; + asset_type := t_row.asset_type; + asset_code := t_row.asset_code; + asset_issuer := t_row.asset_issuer; + created_at := t_row.created_at; + issued := (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type = 'account_debited' AND account = t_row.asset_issuer AND effect_id <= t_row.effect_id); + transferred := (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type = 'account_debited' AND account != t_row.asset_issuer AND effect_id <= t_row.effect_id); + accounts_with_trustline := (SELECT COUNT(DISTINCT account) FROM effects WHERE account != t_row.asset_issuer AND type = 'trustline_created' AND effect_id <= t_row.effect_id); + accounts_with_payments := (SELECT COUNT(DISTINCT account) FROM effects WHERE account != t_row.asset_issuer AND type = 'account_debited' AND effect_id <= t_row.effect_id); + payments := (SELECT COUNT(*) FROM effects WHERE type = 'account_debited' AND account != t_row.asset_issuer AND effect_id <= t_row.effect_id); + RETURN NEXT; + END LOOP; +END; +$$ +LANGUAGE plpgsql; + +DROP TABLE asset_stats; +DROP FUNCTION repopulate_asset_stats; diff --git a/models/asset_stat/asset_stat.go b/models/asset_stat/asset_stat.go index 2df97a5..a63d9a1 100644 --- a/models/asset_stat/asset_stat.go +++ b/models/asset_stat/asset_stat.go @@ -4,12 +4,11 @@ import ( "github.com/cndy-store/analytics/utils/bigint" "github.com/cndy-store/analytics/utils/filter" "github.com/cndy-store/analytics/utils/sql" - "github.com/stellar/go/clients/horizon" "time" ) type AssetStat struct { - Id *uint32 `db:"id" json:"-"` + EffectId *string `db:"effect_id", json:"effect_id"` // TODO: omitempty, also add to tests? PagingToken *string `db:"paging_token" json:"paging_token,omitempty"` AssetType *string `db:"asset_type" json:"asset_type,omitempty"` AssetCode *string `db:"asset_code" json:"asset_code,omitempty"` @@ -29,26 +28,26 @@ type AssetStat struct { JsonTransferred *string `db:"-" json:"transferred"` } -func New(db sql.Database, effect horizon.Effect, timestamp time.Time) (err error) { - // Store amount_transfered and amount_issued upon insert in a different table - // (analogue to the asset endpoint of Horizon) +type Filter struct { + From *time.Time + To *time.Time +} - _, err = db.Exec(`INSERT INTO asset_stats(paging_token, asset_code, asset_issuer, asset_type, created_at, issued, transferred, accounts_with_trustline, accounts_with_payments, payments) - VALUES ($1, $2, $3, $4, $5, - (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type='account_debited' AND account=$6), - (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type='account_debited' AND account!=$6), - (SELECT COUNT(DISTINCT account) FROM effects WHERE type='trustline_created' AND account!=$6), - (SELECT COUNT(DISTINCT account) FROM effects WHERE type='account_debited' AND account!=$6), - (SELECT COUNT(*) FROM effects WHERE type='account_debited' AND account!=$6) - )`, - effect.PT, effect.Asset.Code, effect.Asset.Issuer, effect.Asset.Type, timestamp, effect.Asset.Issuer) +func (f *Filter) Defaults() { + if f.From == nil { + t := time.Unix(0, 0) + f.From = &t + } - return + if f.To == nil { + t := time.Now() + f.To = &t + } } func Get(db sql.Database, filter filter.Filter) (stats []AssetStat, err error) { filter.Defaults() - err = db.Select(&stats, `SELECT * FROM asset_stats WHERE asset_code=$1 AND asset_issuer=$2 AND created_at BETWEEN $3::timestamp AND $4::timestamp ORDER BY id`, + err = db.Select(&stats, `SELECT * FROM asset_stats($1, $2) WHERE created_at BETWEEN $3::timestamp AND $4::timestamp ORDER BY effect_id`, filter.AssetCode, filter.AssetIssuer, filter.From, filter.To) if err == sql.ErrNoRows { err = nil @@ -68,7 +67,7 @@ func Get(db sql.Database, filter filter.Filter) (stats []AssetStat, err error) { func Latest(db sql.Database, filter filter.Filter) (stats AssetStat, err error) { filter.Defaults() - err = db.Get(&stats, `SELECT * FROM asset_stats WHERE asset_code=$1 AND asset_issuer=$2 ORDER BY id DESC LIMIT 1`, + err = db.Get(&stats, `SELECT * FROM asset_stats($1, $2) ORDER BY effect_id DESC LIMIT 1`, filter.AssetCode, filter.AssetIssuer) if err == sql.ErrNoRows { err = nil diff --git a/models/effect/effect.go b/models/effect/effect.go index 852e903..8689820 100644 --- a/models/effect/effect.go +++ b/models/effect/effect.go @@ -2,7 +2,6 @@ package effect import ( "encoding/json" - "github.com/cndy-store/analytics/models/asset_stat" "github.com/cndy-store/analytics/utils/bigint" "github.com/cndy-store/analytics/utils/filter" "github.com/cndy-store/analytics/utils/sql" @@ -109,12 +108,6 @@ func New(db sql.Database, effect horizon.Effect) (err error) { return } - // Store asset stats upon insert in a different table - err = assetStat.New(db, effect, operation.CreatedAt) - if err != nil { - return - } - log.Printf("--+--[ %s ]", effect.Asset.Code) log.Printf(" |") log.Printf(" +-> Type: %s", effect.Type) diff --git a/utils/test/test.go b/utils/test/test.go index 515c477..96c070a 100644 --- a/utils/test/test.go +++ b/utils/test/test.go @@ -57,10 +57,5 @@ func InsertTestData(tx *sqlx.Tx) (err error) { } } - _, err = tx.Exec(`SELECT repopulate_asset_stats()`) - if err != nil { - return - } - return }