From 29fc96c0405ad8b4fb083ef2198569fdb94f8834 Mon Sep 17 00:00:00 2001 From: Louis Schmieder Date: Thu, 26 May 2022 21:53:09 +0200 Subject: [PATCH] orm: document & fix pg (#14533) --- vlib/orm/orm.v | 82 ++++++++++++++++++++++++++++++++++----------- vlib/orm/orm_test.v | 19 ++++++++--- vlib/pg/orm.v | 48 ++++++++++++++------------ vlib/sqlite/orm.v | 8 +++++ 4 files changed, 112 insertions(+), 45 deletions(-) diff --git a/vlib/orm/orm.v b/vlib/orm/orm.v index baad8adb54..4e5d62633f 100644 --- a/vlib/orm/orm.v +++ b/vlib/orm/orm.v @@ -102,6 +102,10 @@ fn (kind OrderType) to_str() string { } } +// Examples for QueryData in SQL: abc == 3 && b == 'test' +// => fields[abc, b]; data[3, 'test']; types[index of int, index of string]; kinds[.eq, .eq]; is_and[true]; +// Every field, data, type & kind of operation in the expr share the same index in the arrays +// is_and defines how they're addicted to each other either and or or pub struct QueryData { pub: fields []string @@ -128,6 +132,17 @@ pub: attrs []StructAttribute } +// table - Table name +// is_count - Either the data will be returned or an integer with the count +// has_where - Select all or use a where expr +// has_order - Order the results +// order - Name of the column which will be ordered +// order_type - Type of order (asc, desc) +// has_limit - Limits the output data +// primary - Name of the primary field +// has_offset - Add an offset to the result +// fields - Fields to select +// types - Types to select pub struct SelectConfig { pub: table string @@ -143,6 +158,14 @@ pub: types []int } +// Interfaces gets called from the backend and can be implemented +// Since the orm supports arrays aswell, they have to be returned too. +// A row is represented as []Primitive, where the data is connected to the fields of the struct by their +// index. The indices are mapped with the SelectConfig.field array. This is the mapping for a struct. +// To have an array, there has to be an array of structs, basically [][]Primitive +// +// Every function without last_id() returns an optional, which returns an error if present +// last_id returns the last inserted id of the db pub interface Connection { @select(config SelectConfig, data QueryData, where QueryData) ?[][]Primitive insert(table string, data QueryData) ? @@ -153,7 +176,12 @@ pub interface Connection { last_id() Primitive } -pub fn orm_stmt_gen(table string, para string, kind StmtKind, num bool, qm string, start_pos int, data QueryData, where QueryData) string { +// Generates an sql stmt, from universal parameter +// q - The quotes character, which can be different in every type, so it's variable +// num - Stmt uses nums at prepared statements (? or ?1) +// qm - Character for prepared statment, qm because of quotation mark like in sqlite +// start_pos - When num is true, it's the start position of the counter +pub fn orm_stmt_gen(table string, q string, kind StmtKind, num bool, qm string, start_pos int, data QueryData, where QueryData) string { mut str := '' mut c := start_pos @@ -163,7 +191,7 @@ pub fn orm_stmt_gen(table string, para string, kind StmtKind, num bool, qm strin mut values := []string{} for _ in 0 .. data.fields.len { - // loop over the length of data.field and generate ?0, ?1 or just ? based on the $num parameter for value placeholders + // loop over the length of data.field and generate ?0, ?1 or just ? based on the $num qmeter for value placeholders if num { values << '$qm$c' c++ @@ -172,16 +200,16 @@ pub fn orm_stmt_gen(table string, para string, kind StmtKind, num bool, qm strin } } - str += 'INSERT INTO $para$table$para (' - str += data.fields.map('$para$it$para').join(', ') + str += 'INSERT INTO $q$table$q (' + str += data.fields.map('$q$it$q').join(', ') str += ') VALUES (' str += values.join(', ') str += ')' } .update { - str += 'UPDATE $para$table$para SET ' + str += 'UPDATE $q$table$q SET ' for i, field in data.fields { - str += '$para$field$para = ' + str += '$q$field$q = ' if data.data.len > i { d := data.data[i] if d is InfixType { @@ -217,12 +245,12 @@ pub fn orm_stmt_gen(table string, para string, kind StmtKind, num bool, qm strin str += ' WHERE ' } .delete { - str += 'DELETE FROM $para$table$para WHERE ' + str += 'DELETE FROM $q$table$q WHERE ' } } if kind == .update || kind == .delete { for i, field in where.fields { - str += '$para$field$para ${where.kinds[i].to_str()} $qm' + str += '$q$field$q ${where.kinds[i].to_str()} $qm' if num { str += '$c' c++ @@ -236,28 +264,32 @@ pub fn orm_stmt_gen(table string, para string, kind StmtKind, num bool, qm strin return str } -pub fn orm_select_gen(orm SelectConfig, para string, num bool, qm string, start_pos int, where QueryData) string { +// Generates an sql select stmt, from universal parameter +// orm - See SelectConfig +// q, num, qm, start_pos - see orm_stmt_gen +// where - See QueryData +pub fn orm_select_gen(orm SelectConfig, q string, num bool, qm string, start_pos int, where QueryData) string { mut str := 'SELECT ' if orm.is_count { str += 'COUNT(*)' } else { for i, field in orm.fields { - str += '$para$field$para' + str += '$q$field$q' if i < orm.fields.len - 1 { str += ', ' } } } - str += ' FROM $para$orm.table$para' + str += ' FROM $q$orm.table$q' mut c := start_pos if orm.has_where { str += ' WHERE ' for i, field in where.fields { - str += '$para$field$para ${where.kinds[i].to_str()} $qm' + str += '$q$field$q ${where.kinds[i].to_str()} $qm' if num { str += '$c' c++ @@ -276,7 +308,7 @@ pub fn orm_select_gen(orm SelectConfig, para string, num bool, qm string, start_ // ordering is *slow*, especially if there are no indexes! if orm.has_order { str += ' ORDER BY ' - str += '$para$orm.order$para ' + str += '$q$orm.order$q ' str += orm.order_type.to_str() } @@ -300,11 +332,19 @@ pub fn orm_select_gen(orm SelectConfig, para string, num bool, qm string, start_ return str } -pub fn orm_table_gen(table string, para string, defaults bool, def_unique_len int, fields []TableField, sql_from_v fn (int) ?string, alternative bool) ?string { - mut str := 'CREATE TABLE IF NOT EXISTS $para$table$para (' +// Generates an sql table stmt, from universal parameter +// table - Table name +// q - see orm_stmt_gen +// defaults - enables default values in stmt +// def_unique_len - sets default unique length for texts +// fields - See TableField +// sql_from_v - Function which maps type indices to sql type names +// alternative - Needed for msdb +pub fn orm_table_gen(table string, q string, defaults bool, def_unique_len int, fields []TableField, sql_from_v fn (int) ?string, alternative bool) ?string { + mut str := 'CREATE TABLE IF NOT EXISTS $q$table$q (' if alternative { - str = 'IF NOT EXISTS (SELECT * FROM sysobjects WHERE name=$para$table$para and xtype=${para}U$para) CREATE TABLE $para$table$para (' + str = 'IF NOT EXISTS (SELECT * FROM sysobjects WHERE name=$q$table$q and xtype=${q}U$q) CREATE TABLE $q$table$q (' } mut fs := []string{} @@ -368,7 +408,7 @@ pub fn orm_table_gen(table string, para string, defaults bool, def_unique_len in if ctyp == '' { return error('Unknown type ($field.typ) for field $field.name in struct $table') } - stmt = '$para$field_name$para $ctyp' + stmt = '$q$field_name$q $ctyp' if defaults && field.default_val != '' { stmt += ' DEFAULT $field.default_val' } @@ -376,7 +416,7 @@ pub fn orm_table_gen(table string, para string, defaults bool, def_unique_len in stmt += ' NOT NULL' } if is_unique { - mut f := 'UNIQUE($para$field_name$para' + mut f := 'UNIQUE($q$field_name$q' if ctyp == 'TEXT' && def_unique_len > 0 { if unique_len > 0 { f += '($unique_len)' @@ -396,18 +436,19 @@ pub fn orm_table_gen(table string, para string, defaults bool, def_unique_len in for k, v in unique { mut tmp := []string{} for f in v { - tmp << '$para$f$para' + tmp << '$q$f$q' } fs << '/* $k */UNIQUE(${tmp.join(', ')})' } } - fs << 'PRIMARY KEY($para$primary$para)' + fs << 'PRIMARY KEY($q$primary$q)' fs << unique_fields str += fs.join(', ') str += ');' return str } +// Get's the sql field type fn sql_field_type(field TableField) int { mut typ := field.typ if field.is_time { @@ -426,6 +467,7 @@ fn sql_field_type(field TableField) int { return typ } +// Get's the sql field name fn sql_field_name(field TableField) string { mut name := field.name for attr in field.attrs { diff --git a/vlib/orm/orm_test.v b/vlib/orm/orm_test.v index 5022e6d52d..d6c20f418a 100644 --- a/vlib/orm/orm_test.v +++ b/vlib/orm/orm_test.v @@ -1,7 +1,7 @@ // import os -// import pg // import term import time +// import pg import sqlite struct Module { @@ -31,9 +31,12 @@ struct TestTime { create time.Time } -fn test_orm_sqlite() { +fn test_orm() { db := sqlite.connect(':memory:') or { panic(err) } - db.exec('drop table if exists User') + // db.exec('drop table if exists User') + + // db := pg.connect(host: 'localhost', port: 5432, user: 'louis', password: 'abc', dbname: 'orm') or { panic(err) } + sql db { create table Module } @@ -242,7 +245,7 @@ fn test_orm_sqlite() { // offset_const := 2 z := sql db { - select from User limit 2 offset offset_const + select from User order by id limit 2 offset offset_const } assert z.len == 2 assert z[0].id == 3 @@ -264,6 +267,7 @@ fn test_orm_sqlite() { } assert updated_oldest.age == 31 + // Remove this when pg is used db.exec('insert into User (name, age) values (NULL, 31)') null_user := sql db { select from User where id == 5 @@ -336,11 +340,18 @@ fn test_orm_sqlite() { sql db { update Module set created = t where id == 1 } + updated_time_mod := sql db { select from Module where id == 1 } + // Note: usually updated_time_mod.created != t, because t has // its microseconds set, while the value retrieved from the DB // has them zeroed, because the db field resolution is seconds. assert updated_time_mod.created.format_ss() == t.format_ss() + + sql db { + drop table Module + drop table TestTime + } } diff --git a/vlib/pg/orm.v b/vlib/pg/orm.v index aeed948b7a..70a4c363f4 100644 --- a/vlib/pg/orm.v +++ b/vlib/pg/orm.v @@ -8,9 +8,13 @@ import net.conv pub fn (db DB) @select(config orm.SelectConfig, data orm.QueryData, where orm.QueryData) ?[][]orm.Primitive { query := orm.orm_select_gen(config, '"', true, '$', 1, where) + + res := pg_stmt_worker(db, query, where, data)? + mut ret := [][]orm.Primitive{} - res := pg_stmt_worker(db, query, orm.QueryData{}, where)? + if config.is_count { + } for row in res { mut row_data := []orm.Primitive{} @@ -166,7 +170,9 @@ fn pg_stmt_match(mut types []u32, mut vals []&char, mut lens []int, mut formats } time.Time { types << u32(Oid.t_int4) - vals << &char(&int(data.unix)) + unix := int(data.unix) + num := conv.htn32(unsafe { &u32(&unix) }) + vals << &char(&num) lens << int(sizeof(u32)) formats << 1 } @@ -178,19 +184,22 @@ fn pg_stmt_match(mut types []u32, mut vals []&char, mut lens []int, mut formats fn pg_type_from_v(typ int) ?string { str := match typ { - 6, 10 { + orm.type_idx['i8'], orm.type_idx['i16'], orm.type_idx['byte'], orm.type_idx['u16'] { 'SMALLINT' } - 7, 11, orm.time { + orm.type_idx['bool'] { + 'BOOLEAN' + } + orm.type_idx['int'], orm.type_idx['u32'], orm.time { 'INT' } - 8, 12 { + orm.type_idx['i64'], orm.type_idx['u64'] { 'BIGINT' } - 13 { + orm.float[0] { 'REAL' } - 14 { + orm.float[1] { 'DOUBLE PRECISION' } orm.string { @@ -212,54 +221,51 @@ fn pg_type_from_v(typ int) ?string { fn str_to_primitive(str string, typ int) ?orm.Primitive { match typ { // bool - 16 { - return orm.Primitive(str.i8() == 1) - } - 18 { + orm.type_idx['bool'] { return orm.Primitive(str == 't') } // i8 - 5 { + orm.type_idx['i8'] { return orm.Primitive(str.i8()) } // i16 - 6 { + orm.type_idx['i16'] { return orm.Primitive(str.i16()) } // int - 7 { + orm.type_idx['int'] { return orm.Primitive(str.int()) } // i64 - 8 { + orm.type_idx['i64'] { return orm.Primitive(str.i64()) } // byte - 9 { + orm.type_idx['byte'] { data := str.i8() return orm.Primitive(*unsafe { &u8(&data) }) } // u16 - 10 { + orm.type_idx['u16'] { data := str.i16() return orm.Primitive(*unsafe { &u16(&data) }) } // u32 - 11 { + orm.type_idx['u32'] { data := str.int() return orm.Primitive(*unsafe { &u32(&data) }) } // u64 - 12 { + orm.type_idx['u64'] { data := str.i64() return orm.Primitive(*unsafe { &u64(&data) }) } // f32 - 13 { + orm.type_idx['f32'] { return orm.Primitive(str.f32()) } // f64 - 14 { + orm.type_idx['f64'] { return orm.Primitive(str.f64()) } orm.string { diff --git a/vlib/sqlite/orm.v b/vlib/sqlite/orm.v index 8aa6720c66..e01b2d369f 100644 --- a/vlib/sqlite/orm.v +++ b/vlib/sqlite/orm.v @@ -6,6 +6,7 @@ import time // sql expr pub fn (db DB) @select(config orm.SelectConfig, data orm.QueryData, where orm.QueryData) ?[][]orm.Primitive { + // 1. Create query and bind necessary data query := orm.orm_select_gen(config, '`', true, '?', 1, where) stmt := db.new_init_stmt(query)? mut c := 1 @@ -19,6 +20,7 @@ pub fn (db DB) @select(config orm.SelectConfig, data orm.QueryData, where orm.Qu mut ret := [][]orm.Primitive{} if config.is_count { + // 2. Get count of returned values & add it to ret array step := stmt.step() if step !in [sqlite_row, sqlite_ok, sqlite_done] { return db.error_message(step, query) @@ -28,6 +30,7 @@ pub fn (db DB) @select(config orm.SelectConfig, data orm.QueryData, where orm.Qu return ret } for { + // 2. Parse returned values step := stmt.step() if step == sqlite_done { break @@ -83,6 +86,7 @@ pub fn (db DB) drop(table string) ? { // helper +// Executes query and bind prepared statement data directly fn sqlite_stmt_worker(db DB, query string, data orm.QueryData, where orm.QueryData) ? { stmt := db.new_init_stmt(query)? mut c := 1 @@ -92,6 +96,7 @@ fn sqlite_stmt_worker(db DB, query string, data orm.QueryData, where orm.QueryDa stmt.finalize() } +// Binds all values of d in the prepared statement fn sqlite_stmt_binder(stmt Stmt, d orm.QueryData, query string, mut c &int) ? { for data in d.data { err := bind(stmt, c, data) @@ -103,6 +108,7 @@ fn sqlite_stmt_binder(stmt Stmt, d orm.QueryData, query string, mut c &int) ? { } } +// Universal bind function fn bind(stmt Stmt, c &int, data orm.Primitive) int { mut err := 0 match data { @@ -128,6 +134,7 @@ fn bind(stmt Stmt, c &int, data orm.Primitive) int { return err } +// Selects column in result and converts it to an orm.Primitive fn (stmt Stmt) sqlite_select_column(idx int, typ int) ?orm.Primitive { mut primitive := orm.Primitive(0) @@ -149,6 +156,7 @@ fn (stmt Stmt) sqlite_select_column(idx int, typ int) ?orm.Primitive { return primitive } +// Convert type int to sql type string fn sqlite_type_from_v(typ int) ?string { return if typ in orm.nums || typ < 0 || typ in orm.num64 || typ == orm.time { 'INTEGER'