From 1943da54a571e596a3224827715e330678497c01 Mon Sep 17 00:00:00 2001 From: Louis Schmieder Date: Sat, 24 Jul 2021 19:49:40 +0200 Subject: [PATCH] orm: integrate psql to orm (#10933) --- cmd/tools/vtest-self.v | 1 + examples/database/orm.v | 12 +- vlib/net/conv/conv.c.v | 21 +++ vlib/net/conv/conv_default.c.v | 46 ++++++ vlib/net/conv/conv_windows.c.v | 21 +++ vlib/pg/oid.v | 171 +++++++++++++++++++++ vlib/pg/orm.v | 272 +++++++++++++++++++++++++++++++++ vlib/pg/pg_orm_test.v | 77 ++++++++++ vlib/v/gen/c/sql.v | 7 +- 9 files changed, 619 insertions(+), 9 deletions(-) create mode 100644 vlib/net/conv/conv.c.v create mode 100644 vlib/net/conv/conv_default.c.v create mode 100644 vlib/net/conv/conv_windows.c.v create mode 100644 vlib/pg/oid.v create mode 100644 vlib/pg/orm.v create mode 100644 vlib/pg/pg_orm_test.v diff --git a/cmd/tools/vtest-self.v b/cmd/tools/vtest-self.v index 90df76de2e..99cc83cbc5 100644 --- a/cmd/tools/vtest-self.v +++ b/cmd/tools/vtest-self.v @@ -10,6 +10,7 @@ const ( skip_test_files = [ 'vlib/context/deadline_test.v' /* sometimes blocks */, 'vlib/mysql/mysql_orm_test.v' /* mysql not installed */, + 'vlib/pg/pg_orm_test.v' /* pg not installed */, ] skip_fsanitize_too_slow = [ // These tests are too slow to be run in the CI on each PR/commit diff --git a/examples/database/orm.v b/examples/database/orm.v index 7a5cd23fd5..a7db00c160 100644 --- a/examples/database/orm.v +++ b/examples/database/orm.v @@ -1,6 +1,6 @@ import sqlite import mysql -// import pg +import pg [table: 'modules'] struct Module { @@ -33,11 +33,11 @@ struct Child { fn main() { sqlite3_array() mysql_array() - // psql_array() + psql_array() sqlite3() mysql() - // psql() + psql() } fn sqlite3_array() { @@ -118,7 +118,6 @@ fn mysql_array() { db.close() } -/* fn psql_array() { mut db := pg.connect(host: 'localhost', user: 'test', password: 'abc', dbname: 'test') or { panic(err) @@ -156,7 +155,7 @@ fn psql_array() { } db.close() -}*/ +} fn sqlite3() { mut db := sqlite.connect(':memory:') or { panic(err) } @@ -225,7 +224,6 @@ fn mysql() { conn.close() } -/* fn psql() { mut db := pg.connect(host: 'localhost', user: 'test', password: 'abc', dbname: 'test') or { panic(err) @@ -259,4 +257,4 @@ fn psql() { eprintln(modul) db.close() -}*/ +} diff --git a/vlib/net/conv/conv.c.v b/vlib/net/conv/conv.c.v new file mode 100644 index 0000000000..e29741a13b --- /dev/null +++ b/vlib/net/conv/conv.c.v @@ -0,0 +1,21 @@ +module conv + +// host to net 32 (htonl) +pub fn htn32(host &u32) u32 { + return C.htonl(host) +} + +// host to net 16 (htons) +pub fn htn16(host &u16) u16 { + return C.htons(host) +} + +// net to host 32 (ntohl) +pub fn nth32(host &u32) u32 { + return C.ntohl(host) +} + +// net to host 16 (ntohs) +pub fn nth16(host &u16) u16 { + return C.ntohs(host) +} diff --git a/vlib/net/conv/conv_default.c.v b/vlib/net/conv/conv_default.c.v new file mode 100644 index 0000000000..8e8c5829e5 --- /dev/null +++ b/vlib/net/conv/conv_default.c.v @@ -0,0 +1,46 @@ +module conv + +#include + +fn C.htonl(host u32) u32 +fn C.htons(host u16) u16 + +fn C.ntohl(net u32) u32 +fn C.ntohs(net u16) u16 + +struct Bytes { +mut: + first u32 + last u32 +} + +union LongLong { + Bytes + ll u64 +} + +// host to net 64 (htonll) +pub fn htn64(host &u64) u64 { + mut ll := LongLong{ + ll: host + } + + unsafe { + ll.first = htn32(ll.first) + ll.last = htn32(ll.last) + } + return unsafe { ll.ll } +} + +// net to host 64 (ntohll) +pub fn nth64(net &u64) u64 { + mut ll := LongLong{ + ll: net + } + + unsafe { + ll.first = nth32(ll.first) + ll.last = nth32(ll.last) + } + return unsafe { ll.ll } +} diff --git a/vlib/net/conv/conv_windows.c.v b/vlib/net/conv/conv_windows.c.v new file mode 100644 index 0000000000..15827f7b25 --- /dev/null +++ b/vlib/net/conv/conv_windows.c.v @@ -0,0 +1,21 @@ +module conv + +#include + +fn C.htonll(host u64) u64 +fn C.htonl(host u32) u32 +fn C.htons(host u16) u16 + +fn C.ntohll(net u32) u32 +fn C.ntohl(net u32) u32 +fn C.ntohs(net u16) u16 + +// host to net 64 (htonll) +pub fn htn64(host &u64) u64 { + return C.htonll(host) +} + +// net to host 64 (htonll) +pub fn nth64(host &u64) u64 { + return C.ntohll(host) +} diff --git a/vlib/pg/oid.v b/vlib/pg/oid.v new file mode 100644 index 0000000000..2f8004d888 --- /dev/null +++ b/vlib/pg/oid.v @@ -0,0 +1,171 @@ +module pg + +pub enum Oid { + t_bool = 16 + t_bytea = 17 + t_char = 18 + t_name = 19 + t_int8 = 20 + t_int2 = 21 + t_int2vector = 22 + t_int4 = 23 + t_regproc = 24 + t_text = 25 + t_oid = 26 + t_tid = 27 + t_xid = 28 + t_cid = 29 + t_vector = 30 + t_pg_ddl_command = 32 + t_pg_type = 71 + t_pg_attribute = 75 + t_pg_proc = 81 + t_pg_class = 83 + t_json = 114 + t_xml = 142 + t__xml = 143 + t_pg_node_tree = 194 + t__json = 199 + t_smgr = 210 + t_index_am_handler = 325 + t_point = 600 + t_lseg = 601 + t_path = 602 + t_box = 603 + t_polygon = 604 + t_line = 628 + t__line = 629 + t_cidr = 650 + t__cidr = 651 + t_float4 = 700 + t_float8 = 701 + t_abstime = 702 + t_reltime = 703 + t_tinterval = 704 + t_unknown = 705 + t_circle = 718 + t__circle = 719 + t_money = 790 + t__money = 791 + t_macaddr = 829 + t_inet = 869 + t__bool = 1000 + t__bytea = 1001 + t__char = 1002 + t__name = 1003 + t__int2 = 1005 + t__int2vector = 1006 + t__int4 = 1007 + t__regproc = 1008 + t__text = 1009 + t__tid = 1010 + t__xid = 1011 + t__cid = 1012 + t__vector = 1013 + t__bpchar = 1014 + t__varchar = 1015 + t__int8 = 1016 + t__point = 1017 + t__lseg = 1018 + t__path = 1019 + t__box = 1020 + t__float4 = 1021 + t__float8 = 1022 + t__abstime = 1023 + t__reltime = 1024 + t__tinterval = 1025 + t__polygon = 1027 + t__ = 1028 + t_aclitem = 1033 + t__aclitem = 1034 + t__macaddr = 1040 + t__inet = 1041 + t_bpchar = 1042 + t_varchar = 1043 + t_date = 1082 + t_time = 1083 + t_timestamp = 1114 + t__timestamp = 1115 + t__date = 1182 + t__time = 1183 + t_timestamptz = 1184 + t__timestamptz = 1185 + t_interval = 1186 + t__interval = 1187 + t__numeric = 1231 + t_pg_database = 1248 + t__cstring = 1263 + t_timetz = 1266 + t__timetz = 1270 + t_bit = 1560 + t__bit = 1561 + t_varbit = 1562 + t__varbit = 1563 + t_numeric = 1700 + t_refcursor = 1790 + t__refcursor = 2201 + t_regprocedure = 2202 + t_regoper = 2203 + t_regoperator = 2204 + t_regclass = 2205 + t_regtype = 2206 + t__regprocedure = 2207 + t__regoper = 2208 + t__regoperator = 2209 + t__regclass = 2210 + t__regtype = 2211 + t_record = 2249 + t_cstring = 2275 + t_any = 2276 + t_anyarray = 2277 + t_v = 2278 + t_trigger = 2279 + t_language_handler = 2280 + t_internal = 2281 + t_opaque = 2282 + t_anyelement = 2283 + t__record = 2287 + t_anynonarray = 2776 + t_pg_authid = 2842 + t_pg_auth_members = 2843 + t__txid_snapshot = 2949 + t_uuid = 2950 + t__uuid = 2951 + t_txid_snapshot = 2970 + t_fdw_handler = 3115 + t_pg_lsn = 3220 + t__pg_lsn = 3221 + t_tsm_handler = 3310 + t_anyenum = 3500 + t_tsvector = 3614 + t_tsquery = 3615 + t_gtsvector = 3642 + t__tsvector = 3643 + t__gtsvector = 3644 + t__tsquery = 3645 + t_regconfig = 3734 + t__regconfig = 3735 + t_regdictionary = 3769 + t__regdictionary = 3770 + t_jsonb = 3802 + t__jsonb = 3807 + t_anyrange = 3831 + t_event_trigger = 3838 + t_int4range = 3904 + t__int4range = 3905 + t_numrange = 3906 + t__numrange = 3907 + t_tsrange = 3908 + t__tsrange = 3909 + t_tstzrange = 3910 + t__tstzrange = 3911 + t_daterange = 3912 + t__daterange = 3913 + t_int8range = 3926 + t__int8range = 3927 + t_pg_shseclabel = 4066 + t_regnamespace = 4089 + t__regnamespace = 4090 + t_regrole = 4096 + t__regrole = 4097 +} diff --git a/vlib/pg/orm.v b/vlib/pg/orm.v new file mode 100644 index 0000000000..f40e643faa --- /dev/null +++ b/vlib/pg/orm.v @@ -0,0 +1,272 @@ +module pg + +import orm +import time +import net.conv + +// sql expr + +pub fn (db DB) @select(config orm.SelectConfig, data orm.QueryData, where orm.QueryData) ?[][]orm.Primitive { + query := orm.orm_select_gen(config, '"', true, '$', 1, where) + mut ret := [][]orm.Primitive{} + + res := pg_stmt_worker(db, query, orm.QueryData{}, where) ? + + for row in res { + mut row_data := []orm.Primitive{} + for i, val in row.vals { + field := str_to_primitive(val, config.types[i]) ? + row_data << field + } + ret << row_data + } + + return ret +} + +// sql stmt + +pub fn (db DB) insert(table string, data orm.QueryData) ? { + query := orm.orm_stmt_gen(table, '"', .insert, true, '$', 1, data, orm.QueryData{}) + pg_stmt_worker(db, query, data, orm.QueryData{}) ? +} + +pub fn (db DB) update(table string, data orm.QueryData, where orm.QueryData) ? { + query := orm.orm_stmt_gen(table, '"', .update, true, '$', 1, data, where) + pg_stmt_worker(db, query, data, where) ? +} + +pub fn (db DB) delete(table string, where orm.QueryData) ? { + query := orm.orm_stmt_gen(table, '"', .delete, true, '$', 1, orm.QueryData{}, where) + pg_stmt_worker(db, query, orm.QueryData{}, where) ? +} + +pub fn (db DB) last_id() orm.Primitive { + query := 'SELECT LASTVAL();' + id := db.q_int(query) or { 0 } + return orm.Primitive(id) +} + +// table + +pub fn (db DB) create(table string, fields []orm.TableField) ? { + query := orm.orm_table_gen(table, '"', true, 0, fields, pg_type_from_v, false) or { return err } + pg_stmt_worker(db, query, orm.QueryData{}, orm.QueryData{}) ? +} + +pub fn (db DB) drop(table string) ? { + query := 'DROP TABLE "$table";' + pg_stmt_worker(db, query, orm.QueryData{}, orm.QueryData{}) ? +} + +// utils + +fn pg_stmt_worker(db DB, query string, data orm.QueryData, where orm.QueryData) ?[]Row { + mut param_types := []u32{} + mut param_vals := []&char{} + mut param_lens := []int{} + mut param_formats := []int{} + + pg_stmt_binder(mut param_types, mut param_vals, mut param_lens, mut param_formats, + data) + pg_stmt_binder(mut param_types, mut param_vals, mut param_lens, mut param_formats, + where) + + res := C.PQexecParams(db.conn, query.str, param_vals.len, param_types.data, param_vals.data, + param_lens.data, param_formats.data, 0) + return db.handle_error_or_result(res, 'orm_stmt_worker') +} + +fn pg_stmt_binder(mut types []u32, mut vals []&char, mut lens []int, mut formats []int, d orm.QueryData) { + for data in d.data { + pg_stmt_match(mut types, mut vals, mut lens, mut formats, data) + } +} + +fn pg_stmt_match(mut types []u32, mut vals []&char, mut lens []int, mut formats []int, data orm.Primitive) { + d := data + match data { + bool { + types << u32(Oid.t_bool) + vals << &char(&(d as bool)) + lens << int(sizeof(bool)) + formats << 1 + } + byte { + types << u32(Oid.t_char) + vals << &char(&(d as byte)) + lens << int(sizeof(byte)) + formats << 1 + } + u16 { + types << u32(Oid.t_int2) + num := conv.htn16(&data) + vals << &char(&num) + lens << int(sizeof(u16)) + formats << 1 + } + u32 { + types << u32(Oid.t_int4) + num := conv.htn32(&data) + vals << &char(&num) + lens << int(sizeof(u32)) + formats << 1 + } + u64 { + types << u32(Oid.t_int8) + num := conv.htn64(&data) + vals << &char(&num) + lens << int(sizeof(u64)) + formats << 1 + } + i8 { + types << u32(Oid.t_char) + vals << &char(&(d as i8)) + lens << int(sizeof(i8)) + formats << 1 + } + i16 { + types << u32(Oid.t_int2) + num := conv.htn16(unsafe { &u16(&data) }) + vals << &char(&num) + lens << int(sizeof(i16)) + formats << 1 + } + int { + types << u32(Oid.t_int4) + num := conv.htn32(unsafe { &u32(&data) }) + vals << &char(&num) + lens << int(sizeof(int)) + formats << 1 + } + i64 { + types << u32(Oid.t_int8) + num := conv.htn64(unsafe { &u64(&data) }) + vals << &char(&num) + lens << int(sizeof(i64)) + formats << 1 + } + f32 { + types << u32(Oid.t_float4) + vals << &char(unsafe { &f32(&(d as f32)) }) + lens << int(sizeof(f32)) + formats << 1 + } + f64 { + types << u32(Oid.t_float8) + vals << &char(unsafe { &f64(&(d as f64)) }) + lens << int(sizeof(f64)) + formats << 1 + } + string { + types << u32(Oid.t_text) + vals << data.str + lens << data.len + formats << 0 + } + time.Time { + types << u32(Oid.t_int4) + vals << &char(&int(data.unix)) + lens << int(sizeof(u32)) + formats << 1 + } + orm.InfixType { + pg_stmt_match(mut types, mut vals, mut lens, mut formats, data.right) + } + } +} + +fn pg_type_from_v(typ int) ?string { + str := match typ { + 6, 10 { + 'SMALLINT' + } + 7, 11 { + 'INT' + } + 8, 12 { + 'BIGINT' + } + 13 { + 'REAL' + } + 14 { + 'DOUBLE PRECISION' + } + orm.string { + 'TEXT' + } + -1 { + 'SERIAL' + } + else { + '' + } + } + if str == '' { + return error('Unknown type $typ') + } + return str +} + +fn str_to_primitive(str string, typ int) ?orm.Primitive { + match typ { + // bool + 16 { + return orm.Primitive(str.i8() == 1) + } + // i8 + 5 { + return orm.Primitive(str.i8()) + } + // i16 + 6 { + return orm.Primitive(str.i16()) + } + // int + 7 { + return orm.Primitive(str.int()) + } + // i64 + 8 { + return orm.Primitive(str.i64()) + } + // byte + 9 { + data := str.i8() + return orm.Primitive(*unsafe { &byte(&data) }) + } + // u16 + 10 { + data := str.i16() + return orm.Primitive(*unsafe { &u16(&data) }) + } + // u32 + 11 { + data := str.int() + return orm.Primitive(*unsafe { &u32(&data) }) + } + // u64 + 12 { + data := str.i64() + return orm.Primitive(*unsafe { &u64(&data) }) + } + // f32 + 13 { + return orm.Primitive(str.f32()) + } + // f64 + 14 { + return orm.Primitive(str.f64()) + } + orm.string { + return orm.Primitive(str) + } + orm.time { + timestamp := str.int() + return orm.Primitive(time.unix(timestamp)) + } + else {} + } + return error('Unknown field type $typ') +} diff --git a/vlib/pg/pg_orm_test.v b/vlib/pg/pg_orm_test.v new file mode 100644 index 0000000000..eb0fd1266b --- /dev/null +++ b/vlib/pg/pg_orm_test.v @@ -0,0 +1,77 @@ +module main + +import orm +import pg + +fn test_pg_orm() { + mut db := pg.connect( + host: 'localhost' + user: 'postgres' + password: '' + dbname: 'postgres' + ) or { panic(err) } + + db.create('Test', [ + orm.TableField{ + name: 'id' + typ: 7 + attrs: [ + StructAttribute{ + name: 'primary' + }, + StructAttribute{ + name: 'sql' + has_arg: true + kind: .plain + arg: 'serial' + }, + ] + }, + orm.TableField{ + name: 'name' + typ: 18 + attrs: [] + }, + orm.TableField{ + name: 'age' + typ: 7 + }, + ]) or { panic(err) } + + db.insert('Test', orm.QueryData{ + fields: ['name', 'age'] + data: [orm.string_to_primitive('Louis'), orm.int_to_primitive(101)] + }) or { panic(err) } + + res := db.@select(orm.SelectConfig{ + table: 'Test' + has_where: true + fields: ['id', 'name', 'age'] + types: [7, 18, 8] + }, orm.QueryData{}, orm.QueryData{ + fields: ['name'] + data: [orm.Primitive('Louis'), i64(101)] + types: [18] + is_and: [true] + kinds: [.eq] + }) or { panic(err) } + + id := res[0][0] + name := res[0][1] + age := res[0][2] + + assert id is int + if id is int { + assert id == 1 + } + + assert name is string + if name is string { + assert name == 'Louis' + } + + assert age is i64 + if age is i64 { + assert age == 101 + } +} diff --git a/vlib/v/gen/c/sql.v b/vlib/v/gen/c/sql.v index b7bd42bca3..68d053cd4c 100644 --- a/vlib/v/gen/c/sql.v +++ b/vlib/v/gen/c/sql.v @@ -32,8 +32,8 @@ fn (mut g Gen) sql_stmt(node ast.SqlStmt) { .mysql { fn_prefix = 'mysql__Connection' } - .mssql { - // g.mssql_create_table(node, typ, expr) + .psql { + fn_prefix = 'pg__DB' } else { verror('This database type `$typ` is not implemented yet in orm') // TODO add better error @@ -507,6 +507,9 @@ fn (mut g Gen) sql_select_expr(node ast.SqlExpr) { .mysql { fn_prefix = 'mysql__Connection' } + .psql { + fn_prefix = 'pg__DB' + } else { verror('This database type `$typ` is not implemented yet in orm') // TODO add better error }