http: remove libcurl dependency; replace it with a simple OpenSSL backend

pull/1487/head
Alexander Medvednikov 2019-08-06 05:54:47 +02:00
parent 69932758db
commit bea8f6d7e5
10 changed files with 203 additions and 411 deletions

View File

@ -1540,8 +1540,14 @@ fn (p mut Parser) name_expr() string {
return p.struct_init(name, is_c_struct_init)
}
}
// C fn
if is_c {
// C const (`C.GLFW_KEY_LEFT`)
if p.peek() != .lpar {
p.gen(name)
p.next()
return 'int'
}
// C fn
f := Fn {
name: name// .replace('c_', '')
is_c: true

View File

@ -234,6 +234,7 @@ fn (p mut Parser) register_global(name, typ string) {
is_const: true
is_global: true
mod: p.mod
is_mut: true
}
}

View File

@ -79,10 +79,12 @@ pub fn print(s string) {
}
__global total_m i64 = 0
//__global nr_mallocs int = 0
pub fn malloc(n int) byteptr {
if n < 0 {
panic('malloc(<0)')
}
//nr_mallocs++
/*
TODO
#ifdef VPLAY

View File

@ -335,19 +335,12 @@ pub fn (s string) right(n int) string {
// substr
pub fn (s string) substr(start, end int) string {
/*
if start > end || start >= s.len || end > s.len || start < 0 || end < 0 {
if start > end || start > s.len || end > s.len || start < 0 || end < 0 {
panic('substr($start, $end) out of bounds (len=$s.len)')
return ''
}
*/
if start >= s.len {
return ''
}
len := end - start
// Copy instead of pointing, like in Java and C#.
// Much easier to free such strings.
mut res := string {
len: len
str: malloc(len + 1)
@ -356,14 +349,14 @@ pub fn (s string) substr(start, end int) string {
res.str[i] = s.str[start + i]
}
res.str[len] = `\0`
return res
/*
res := string {
str: s.str + start
len: len
}
return res
*/
return res
}
// KMP search
@ -576,6 +569,9 @@ pub fn (s string) trim_space() string {
// C.printf('end=%d c=%d %c\n', end, res.str[end])
end--
}
if i > end + 1 {
return s
}
res := s.substr(i, end + 1)
// println('after SPACE "$res"')
return res

View File

@ -9,23 +9,28 @@ import os
type downloadfn fn (written int)
type download_finished_fn fn ()
/*
struct DownloadStruct {
mut:
stream voidptr
written int
cb downloadfn
}
*/
fn download_cb(ptr voidptr, size, nmemb size_t, userp voidptr) int {
fn download_cb(ptr voidptr, size, nmemb size_t, userp voidptr) {
/*
mut data := &DownloadStruct(userp)
written := C.fwrite(ptr, size, nmemb, data.stream)
data.written += written
data.cb(data.written)
//#data->cb(data->written); // TODO
return written
*/
}
fn download_file_with_progress(url, out string, cb downloadfn, cb_finished fn()) {
/*
curl := C.curl_easy_init()
if isnil(curl) {
return
@ -45,10 +50,11 @@ fn download_file_with_progress(url, out string, cb downloadfn, cb_finished fn())
C.curl_easy_cleanup(curl)
C.fclose(fp)
cb_finished()
*/
}
fn download_file(url, out string) {
download_file_with_progress(url, out, empty, empty)
//download_file_with_progress(url, out, empty, empty)
}
fn empty() {

View File

@ -89,3 +89,91 @@ pub fn (req mut Request) add_header(key, val string) {
// req.h = h
}
pub fn (req &Request) do() Response {
mut headers := map[string]string{}
if req.typ == 'POST' {
// req.headers << 'Content-Type: application/x-www-form-urlencoded'
}
for key, val in req.headers {
//h := '$key: $val'
}
mut url := req.url
mut host := url
mut path := '/'
is_ssl := req.url.starts_with('https://')
if !is_ssl {
panic('non https requests are not supported right now')
}
mut pos := url.index('://')
if pos == -1 { return Response{} } //error('ff')}
url = url.right(pos + 3)
pos = url.index('/')
if pos > -1 {
host = url.left(pos)
host = host.clone()
path = url.right(pos)
}
s := ssl_do(req.typ, host, path)
first_header := s.all_before('\n')
mut status_code := 0
if first_header.contains('HTTP/') {
val := first_header.find_between(' ', ' ')
status_code = val.int()
}
mut text := ''
// Build resp headers map and separate the body
mut nl_pos := 3
mut i := 1
for {
old_pos := nl_pos
nl_pos = s.index_after('\n', nl_pos+1)
if nl_pos == -1 {
break
}
h := s.substr(old_pos + 1, nl_pos)
// End of headers
if h.len <= 1 {
text = s.right(nl_pos + 1)
break
}
i++
pos = h.index(':')
if pos == -1 {
continue
}
//if h.contains('Content-Type') {
//continue
//}
key := h.left(pos)
val := h.right(pos + 2)
headers[key] = val.trim_space()
}
return Response {
status_code: status_code
headers: headers
text: text
}
}
pub fn unescape_url(s string) string {
panic('http.unescape_url() was replaced with urllib.query_unescape()')
return ''
}
pub fn escape_url(s string) string {
panic('http.escape_url() was replaced with urllib.query_escape()')
return ''
}
pub fn unescape(s string) string {
panic('http.unescape() was replaced with http.unescape_url()')
return ''
}
pub fn escape(s string) string {
panic('http.escape() was replaced with http.escape_url()')
return ''
}
type wsfn fn (s string, ptr voidptr)

View File

@ -1,212 +0,0 @@
// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module http
#include <curl/curl.h>
#flag darwin -lcurl
#flag windows -lcurl
#flag linux -lcurl
fn C.curl_easy_init() *C.CURL
fn foo() {}
type wsfn fn (s string, ptr voidptr)
struct MemoryStruct {
size size_t
//ws_func wsfn
ws_func fn(string, voidptr)
user_ptr voidptr // for wsfn
strings []string
}
import const (
CURLOPT_WRITEFUNCTION
CURLOPT_SSL_VERIFYPEER
CURLOPT_HEADERFUNCTION
CURLOPT_WRITEDATA
CURLOPT_HEADERDATA
CURLOPT_FOLLOWLOCATION
CURLOPT_URL
CURLOPT_VERBOSE
CURLOPT_HTTP_VERSION
CURL_HTTP_VERSION_1_1
CURLOPT_HTTPHEADER
CURLOPT_POSTFIELDS
CURLOPT_CUSTOMREQUEST
CURLOPT_TCP_KEEPALIVE
CURLINFO_CONTENT_LENGTH_DOWNLOAD
CURLE_OK
)
fn C.curl_easy_strerror(curl voidptr) byteptr
fn C.curl_easy_perform(curl voidptr) int// C.CURLcode
fn write_fn(contents byteptr, size, nmemb int, _mem *MemoryStruct) int {
mut mem := _mem
// # printf("size =%d nmemb=%d contents=%s\n", size, nmemb, contents);
realsize := size * nmemb// TODO size_t ?
/*
if false && !isnil(mem.ws_func) {
//C.printf('\n\nhttp_mac.m: GOT WS FUNC. size=%d\n', realsize)
// Skip negative and 0 junk chars in the WS string
mut start := 0
for i := 0; i < realsize; i++ {
// printf("char=%d %c\n", s[i], s[i]);
if contents[i] == 0 && start == 0 {
start = i
break
}
}
contents += start + 1
C.printf('GOOD CONTEnTS=%s\n', contents)
s := string(contents)
f := mem.ws_func
f(s, mem.user_ptr)
//# mem->ws_func(s, mem->user_ptr);
}
*/
mut c := string(contents)
c = c.trim_space()
// Need to clone because libcurl reuses this memory
mem.strings << c.clone()
return realsize
}
struct C.curl_slist { }
pub fn (req &Request) do() Response {
//println('req.do() mac/linux url="$req.url" data="$req.data"')
// println('req.do() url="$req.url"')
/*
mut resp := Response {
headers: map[string]string{}
}
*/
mut headers := map[string]string{}
// no data at this point
chunk := MemoryStruct {
ws_func: req.ws_func
user_ptr: req.user_ptr
}
// header chunk
hchunk := MemoryStruct {
ws_func: foo
user_ptr: 0
}
// init curl
curl := C.curl_easy_init()
if isnil(curl) {
println('curl init failed')
return Response{}
}
// options
// url2 := req.url.clone()
C.curl_easy_setopt(curl, CURLOPT_URL, req.url.str)// ..clone())
// C.curl_easy_setopt(curl, CURLOPT_URL, 'http://example.com')
// return resp
// curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
$if windows {
C.curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0)
}
C.curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_fn)
C.curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, write_fn)
C.curl_easy_setopt(curl, CURLOPT_WRITEDATA, &chunk)
C.curl_easy_setopt(curl, CURLOPT_HEADERDATA, &hchunk)
C.curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1)
if req.typ == 'POST' {
C.curl_easy_setopt(curl, CURLOPT_POSTFIELDS, req.data.str)
C.curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, 'POST')
// req.headers << 'Content-Type: application/x-www-form-urlencoded'
}
// Add request headers
mut hlist := &C.curl_slist{!}
// for i, h := range req.headers {
for key, val in req.headers {
h := '$key: $val'
hlist = C.curl_slist_append(hlist, h.str)
}
// curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, // (long)CURL_HTTP_VERSION_2TLS);<3B>`C<>ʀ9<CA80>
C.curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1)
if req.verbose {
C.curl_easy_setopt(curl, CURLOPT_VERBOSE, 1)
}
C.curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hlist)
C.curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1)
C.curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1)
//println('bef easy()')
res := C.curl_easy_perform(curl)
//println('after easy()')
if res != CURLE_OK {
err := C.curl_easy_strerror(res)
println('curl_easy_perform() failed: $err')
}
body := chunk.strings.join('')// string(chunk.memory)
// chunk.strings.free()
// resp.headers = hchunk.strings
if hchunk.strings.len == 0 {
return Response{}
}
first_header := hchunk.strings.first()
mut status_code := 0
if first_header.contains('HTTP/') {
val := first_header.find_between(' ', ' ')
status_code = val.int()
}
// Build resp headers map
// println('building resp headers hchunk.strings.len')
for h in hchunk.strings {
// break
// println(h)
vals := h.split(':')
pos := h.index(':')
if pos == -1 {
continue
}
if h.contains('Content-Type') {
continue
}
key := h.left(pos)
val := h.right(pos + 2)
//println('"$h" "$key" *** "$val"')
// val2 := val.trim_space()
// println('val2="$val2"')
headers[key] = val// val.trim_space()
}
C.curl_easy_cleanup(curl)
//println('end of req.do() url="$req.url"')
return Response {
text: body
status_code: status_code
headers: headers
}
}
pub fn unescape_url(s string) string {
return string(byteptr(C.curl_unescape(s.str, s.len)))
}
pub fn escape_url(s string) string {
return string(byteptr(C.curl_escape(s.str, s.len)))
}
pub fn unescape(s string) string {
panic('http.unescape() was replaced with http.unescape_url()')
return ''
}
pub fn escape(s string) string {
panic('http.escape() was replaced with http.escape_url()')
return ''
}
// ////////////////
fn (req &Request) do2() Response {
mut resp := Response{}
return resp
}

View File

@ -1,10 +1,12 @@
import http
import net.urllib
fn test_escape_unescape() {
/*
original := 'те ст: т\\%'
escaped := http.escape_url(original)
escaped := urllib.query_escape(original) or { assert false return}
assert escaped == '%D1%82%D0%B5%20%D1%81%D1%82%3A%20%D1%82%5C%25'
unescaped := http.unescape_url(escaped)
unescaped := urllib.query_unescape(escaped) or { assert false return }
assert unescaped == original
*/
}

View File

@ -1,183 +0,0 @@
// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module http
import time
#flag -lwininet -lShlwapi
#flag -lurlmon
#include <WinInet.h>
#include "urlmon.h"
#include <shlwapi.h>
import const (
INTERNET_OPEN_TYPE_PRECONFIG
INTERNET_DEFAULT_HTTP_PORT
INTERNET_DEFAULT_HTTPS_PORT
INTERNET_SERVICE_HTTP
INTERNET_MAX_URL_LENGTH
URL_ESCAPE_PERCENT
URL_ESCAPE_SEGMENT_ONLY
HTTP_QUERY_RAW_HEADERS_CRLF
// flags
INTERNET_FLAG_HYPERLINK
INTERNET_FLAG_IGNORE_CERT_CN_INVALID
INTERNET_FLAG_IGNORE_CERT_DATE_INVALID
INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTP
INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTPS
INTERNET_FLAG_NO_AUTH
INTERNET_FLAG_NO_CACHE_WRITE
INTERNET_FLAG_NO_UI
INTERNET_FLAG_NO_COOKIES
INTERNET_FLAG_KEEP_CONNECTION
INTERNET_FLAG_PRAGMA_NOCACHE
INTERNET_FLAG_SECURE
INTERNET_FLAG_RELOAD
)
const (
BUF_MAX = 1024
URL_ESCAPE_AS_UTF8 = 0x00040000 // missing in mingw, require Windows 7
URL_ESCAPE_ASCII_URI_COMPONENT = 0x00080000 // missing in mingw, require Windows 8
)
pub fn (req &Request) do() Response {
emptyresp := Response{}
mut url := req.url
//println('\n\nhttp.do() WIN URL="$url" TYP=$req.typ data="$req.data" headers.len=req.headers.len"')
is_ssl := req.url.starts_with('https://')
mut pos := url.index('://')
if pos == -1 {return emptyresp}
url = url.right(pos + 3)
mut host := url
mut path := '/'
pos = url.index('/')
if pos > -1 {
host = url.left(pos)
host = host.clone()
path = url.right(pos)
}
mut headers := ''
mut resp_headers := ''
for key, val in req.headers {
headers += '$key: $val\r\n'
}
if req.typ == 'POST' {
headers += 'Content-Type: application/x-www-form-urlencoded'
}
data := req.data
// Retrieve default http user agent
user_agent := ''
internet := C.InternetOpen(user_agent.to_wide(), INTERNET_OPEN_TYPE_PRECONFIG, 0, 0, 0)
if isnil(internet) {
println('InternetOpen() failed')
return emptyresp
}
port := int(if is_ssl{INTERNET_DEFAULT_HTTPS_PORT} else { INTERNET_DEFAULT_HTTP_PORT})
connect := C.InternetConnect(internet, host.to_wide(), port, 0, 0, INTERNET_SERVICE_HTTP, 0, 0)
if isnil(connect) {
e := C.GetLastError()
println('[windows] InternetConnect() failed')
C.printf('err=%d\n', e)
return emptyresp
}
mut flags :=
INTERNET_FLAG_HYPERLINK | INTERNET_FLAG_IGNORE_CERT_CN_INVALID |
INTERNET_FLAG_IGNORE_CERT_DATE_INVALID |
INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTP |
INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTPS | INTERNET_FLAG_NO_AUTH |
INTERNET_FLAG_NO_CACHE_WRITE | INTERNET_FLAG_NO_UI |
INTERNET_FLAG_NO_COOKIES | // ...
INTERNET_FLAG_KEEP_CONNECTION |
INTERNET_FLAG_PRAGMA_NOCACHE | INTERNET_FLAG_RELOAD
if is_ssl {
flags = flags | INTERNET_FLAG_SECURE
}
request := C.HttpOpenRequest(connect, req.typ.to_wide(), path.to_wide(), 'HTTP/1.1'.to_wide(), 0, 0, flags, 0)
if isnil(request) {
println('HttpOpenRequest() failed')
return emptyresp
}
ret := C.HttpSendRequest(request, headers.to_wide(), -1, data.str, data.len)
// Get response headers
// Todo call twice to get len
size := 1024
h_buf := malloc(size)
C.HttpQueryInfo(request, HTTP_QUERY_RAW_HEADERS_CRLF, h_buf, &size, 0)
// Get response body
mut buf := [1025]byte
mut nr_read := 0
mut s := ''
for {
ok := C.InternetReadFile(request, buf, BUF_MAX, &nr_read)
if !ok {
println('InternetReadFile() not ok ')
}
if ok && nr_read == 0 {
if req.url.contains('websocket') {
println('win sleeping 2')
time.sleep(2)
continue
}
break
}
buf[nr_read] = 0
s += tos(buf, nr_read) // TODO perf
nr_read = 0
}
C.InternetCloseHandle(request)
C.InternetCloseHandle(connect)
C.InternetCloseHandle(internet)
resp_headers = string(h_buf)
hh := resp_headers.split('\n')
mut resp := Response {
text: s
headers: map[string]string{}
// headers: resp_headers
//hard coded http version 'HTTP/1.1' + space - 9
//3-digit HTTP status codes - 9+3
//HTTP/1.1 200 OK
status_code: resp_headers.substr(9,12).int()
}
for h in hh {
hpos := h.index(':')
if hpos == -1 {
continue
}
key := h.left(hpos)
val := h.right(hpos + 1)
// println('$key => $val')
resp.headers[key] = val.trim_space()
}
return resp
}
pub fn escape_url(s string) string {
mut buf := &u16(malloc(INTERNET_MAX_URL_LENGTH * 2)) // INTERNET_MAX_URL_LENGTH * sizeof(wchar_t)
mut nr_chars := INTERNET_MAX_URL_LENGTH
res := C.UrlEscape(s.to_wide(), buf, &nr_chars, URL_ESCAPE_PERCENT | URL_ESCAPE_AS_UTF8 | URL_ESCAPE_ASCII_URI_COMPONENT)
return string_from_wide2(buf, nr_chars)
}
pub fn unescape_url(s string) string {
mut buf := &u16(malloc(INTERNET_MAX_URL_LENGTH * 2))
mut nr_chars := INTERNET_MAX_URL_LENGTH
res := C.UrlUnescape(s.to_wide(), buf, &nr_chars, URL_ESCAPE_AS_UTF8 | URL_ESCAPE_ASCII_URI_COMPONENT)
return string_from_wide2(buf, nr_chars)
}
pub fn unescape(s string) string {
panic('http.unescape() was replaced with http.unescape_url()')
return ''
}
pub fn escape(s string) string {
panic('http.escape() was replaced with http.escape_url()')
return ''
}
fn C.InternetReadFile(voidptr, voidptr, int, intptr) bool

View File

@ -0,0 +1,86 @@
// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module http
import strings
#flag -I @VROOT/thirdparty/openssl/include
#flag -lssl -lcrypto
#include <openssl/ssl.h>
struct C.SSL {
}
fn init_openssl() {
C.SSL_library_init()
C.SSL_load_error_strings()
C.OPENSSL_config(0)
}
fn ssl_do(method, host_name, path string) string {
init_openssl()
ssl_method := C.SSLv23_method()
if isnil(method) {
}
ctx := C.SSL_CTX_new(ssl_method)
if isnil(ctx) {
}
C.SSL_CTX_set_verify_depth(ctx, 4)
flags := C.SSL_OP_NO_SSLv2 | C.SSL_OP_NO_SSLv3 | C.SSL_OP_NO_COMPRESSION
C.SSL_CTX_set_options(ctx, flags)
mut res := C.SSL_CTX_load_verify_locations(ctx, 'random-org-chain.pem', 0)
if res != 1 {
}
web := C.BIO_new_ssl_connect(ctx)
if isnil(ctx) {
}
addr := host_name + ':443'
res = C.BIO_set_conn_hostname(web, addr.str)
if res != 1 {
}
ssl := &C.SSL{!}
C.BIO_get_ssl(web, &ssl)
if isnil(ssl) {
}
preferred_ciphers := 'HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4'
res = C.SSL_set_cipher_list(ssl, preferred_ciphers.str)
if res != 1 {
}
res = C.SSL_set_tlsext_host_name(ssl, host_name.str)
out := C.BIO_new_fp(stdout, C.BIO_NOCLOSE)
res = C.BIO_do_connect(web)
res = C.BIO_do_handshake(web)
cert := C.SSL_get_peer_certificate(ssl)
res = C.SSL_get_verify_result(ssl)
///////
s := '$method $path HTTP/1.1\r\n' +
'Host: $host_name\r\n' +
'Connection: close\r\n\r\n'
C.BIO_puts(web, s.str)
C.BIO_puts(out, '\n')
mut sb := strings.new_builder(100)
for {
buff := [1536]byte
len := int(C.BIO_read(web, buff, 1536) )
if len > 0 {
sb.write(tos(buff, len))
}
else {
break
}
}
if !isnil(out) {
C.BIO_free(out)
}
if !isnil(web) {
C.BIO_free_all(web)
}
if !isnil(ctx) {
C.SSL_CTX_free(ctx)
}
return sb.str()
}