From a80f2ad5ed67c1b91fe6729c17324b33bfa3d991 Mon Sep 17 00:00:00 2001 From: tan Date: Thu, 28 Dec 2023 12:34:52 +0530 Subject: [PATCH 1/5] feat: add TLS support This adds TLS support. An abstraction for the redis transport `RedisTransport` has been added. The existing TCP transport has been moved in as an implementation `TCPTransport <: RedisTransport`. A new TLS transport has been added as `TLSTransport <: RedisTransport`. This can in future be extended to support unix sockets too (ref: #84) fixes: #87 --- Project.toml | 2 + src/Redis.jl | 2 + src/client.jl | 4 +- src/connection.jl | 85 ++-- src/parser.jl | 28 +- src/transport/tcp.jl | 19 + src/transport/tls.jl | 56 +++ src/transport/transport.jl | 33 ++ test/client_tests.jl | 50 +-- test/redis_tests.jl | 792 +++++++++++++++++++------------------ test/runtests.jl | 15 +- test/test_utils.jl | 32 ++ 12 files changed, 650 insertions(+), 468 deletions(-) create mode 100644 src/transport/tcp.jl create mode 100644 src/transport/tls.jl create mode 100644 src/transport/transport.jl create mode 100644 test/test_utils.jl diff --git a/Project.toml b/Project.toml index c95ae39..35bc615 100644 --- a/Project.toml +++ b/Project.toml @@ -6,10 +6,12 @@ version = "2.0.0" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d" [compat] julia = "^1" DataStructures = "^0.18" +MbedTLS = "0.6.8, 0.7, 1" [extras] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" diff --git a/src/Redis.jl b/src/Redis.jl index 1e70660..9ea9529 100644 --- a/src/Redis.jl +++ b/src/Redis.jl @@ -1,6 +1,7 @@ module Redis using Dates using Sockets +using MbedTLS import Base.get, Base.keys, Base.time @@ -59,6 +60,7 @@ export sentinel_masters, sentinel_master, sentinel_slaves, sentinel_getmasteradd export REDIS_PERSISTENT_KEY, REDIS_EXPIRED_KEY include("exceptions.jl") +include("transport/transport.jl") include("connection.jl") include("parser.jl") include("client.jl") diff --git a/src/client.jl b/src/client.jl index 2c58abf..34117e0 100644 --- a/src/client.jl +++ b/src/client.jl @@ -160,8 +160,8 @@ end function subscription_loop(conn::SubscriptionConnection, err_callback::Function) while is_connected(conn) try - l = getline(conn.socket) - reply = parseline(l, conn.socket) + l = getline(conn.transport) + reply = parseline(l, conn.transport) reply = convert_reply(reply) message = SubscriptionMessage(reply) if message.message_type == SubscriptionMessageType.Message diff --git a/src/connection.jl b/src/connection.jl index 04d22a6..6ca94c1 100644 --- a/src/connection.jl +++ b/src/connection.jl @@ -1,5 +1,3 @@ -import Sockets.connect, Sockets.TCPSocket, Base.StatusActive, Base.StatusOpen, Base.StatusPaused - abstract type RedisConnectionBase end abstract type SubscribableConnection<:RedisConnectionBase end @@ -8,7 +6,7 @@ struct RedisConnection <: SubscribableConnection port::Integer password::AbstractString db::Integer - socket::TCPSocket + transport::Transport.RedisTransport end struct SentinelConnection <: SubscribableConnection @@ -16,7 +14,7 @@ struct SentinelConnection <: SubscribableConnection port::Integer password::AbstractString db::Integer - socket::TCPSocket + transport::Transport.RedisTransport end struct TransactionConnection <: RedisConnectionBase @@ -24,7 +22,7 @@ struct TransactionConnection <: RedisConnectionBase port::Integer password::AbstractString db::Integer - socket::TCPSocket + transport::Transport.RedisTransport end mutable struct PipelineConnection <: RedisConnectionBase @@ -32,7 +30,7 @@ mutable struct PipelineConnection <: RedisConnectionBase port::Integer password::AbstractString db::Integer - socket::TCPSocket + transport::Transport.RedisTransport num_commands::Integer end @@ -43,57 +41,83 @@ struct SubscriptionConnection <: RedisConnectionBase db::Integer callbacks::Dict{AbstractString, Function} pcallbacks::Dict{AbstractString, Function} - socket::TCPSocket + transport::Transport.RedisTransport end -function RedisConnection(; host="127.0.0.1", port=6379, password="", db=0) +Transport.get_sslconfig(s::RedisConnectionBase) = Transport.get_sslconfig(s.transport) + +function RedisConnection(; host="127.0.0.1", port=6379, password="", db=0, sslconfig=nothing) try - socket = connect(host, port) - connection = RedisConnection(host, port, password, db, socket) + connection = RedisConnection( + host, + port, + password, + db, + Transport.transport(host, port, sslconfig) + ) on_connect(connection) catch throw(ConnectionException("Failed to connect to Redis server")) end end -function SentinelConnection(; host="127.0.0.1", port=26379, password="", db=0) +function SentinelConnection(; host="127.0.0.1", port=26379, password="", db=0, sslconfig=nothing) try - socket = connect(host, port) - sentinel_connection = SentinelConnection(host, port, password, db, socket) + sentinel_connection = SentinelConnection( + host, + port, + password, + db, + Transport.transport(host, port, sslconfig) + ) on_connect(sentinel_connection) catch throw(ConnectionException("Failed to connect to Redis sentinel")) end end -function TransactionConnection(parent::RedisConnection) +function TransactionConnection(parent::RedisConnection; sslconfig=Transport.get_sslconfig(parent)) try - socket = connect(parent.host, parent.port) - transaction_connection = TransactionConnection(parent.host, - parent.port, parent.password, parent.db, socket) + transaction_connection = TransactionConnection( + parent.host, + parent.port, + parent.password, + parent.db, + Transport.transport(parent.host, parent.port, sslconfig) + ) on_connect(transaction_connection) catch throw(ConnectionException("Failed to create transaction")) end end -function PipelineConnection(parent::RedisConnection) +function PipelineConnection(parent::RedisConnection; sslconfig=Transport.get_sslconfig(parent)) try - socket = connect(parent.host, parent.port) - pipeline_connection = PipelineConnection(parent.host, - parent.port, parent.password, parent.db, socket, 0) + pipeline_connection = PipelineConnection( + parent.host, + parent.port, + parent.password, + parent.db, + Transport.transport(parent.host, parent.port, sslconfig), + 0 + ) on_connect(pipeline_connection) catch throw(ConnectionException("Failed to create pipeline")) end end -function SubscriptionConnection(parent::SubscribableConnection) +function SubscriptionConnection(parent::SubscribableConnection; sslconfig=Transport.get_sslconfig(parent)) try - socket = connect(parent.host, parent.port) - subscription_connection = SubscriptionConnection(parent.host, - parent.port, parent.password, parent.db, Dict{AbstractString, Function}(), - Dict{AbstractString, Function}(), socket) + subscription_connection = SubscriptionConnection( + parent.host, + parent.port, + parent.password, + parent.db, + Dict{AbstractString, Function}(), + Dict{AbstractString, Function}(), + Transport.transport(parent.host, parent.port, sslconfig) + ) on_connect(subscription_connection) catch throw(ConnectionException("Failed to create subscription")) @@ -101,19 +125,16 @@ function SubscriptionConnection(parent::SubscribableConnection) end function on_connect(conn::RedisConnectionBase) - # disable nagle and enable quickack to speed up the usually small exchanges - Sockets.nagle(conn.socket, false) - Sockets.quickack(conn.socket, true) - + Transport.set_props!(conn.transport) conn.password != "" && auth(conn, conn.password) conn.db != 0 && select(conn, conn.db) conn end function disconnect(conn::RedisConnectionBase) - close(conn.socket) + Transport.close(conn.transport) end function is_connected(conn::RedisConnectionBase) - conn.socket.status == StatusActive || conn.socket.status == StatusOpen || conn.socket.status == StatusPaused + Transport.is_connected(conn.transport) end diff --git a/src/parser.jl b/src/parser.jl index 8796337..99b6200 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -1,8 +1,8 @@ """ Formatting of incoming Redis Replies """ -function getline(s::TCPSocket) - l = chomp(readline(s)) +function getline(t::Transport.RedisTransport) + l = chomp(Transport.read_line(t)) length(l) > 1 || throw(ProtocolException("Invalid response received: $l")) return l end @@ -12,15 +12,15 @@ convert_reply(reply::Array) = [convert_reply(r) for r in reply] convert_reply(x) = x function read_reply(conn::RedisConnectionBase) - l = getline(conn.socket) - reply = parseline(l, conn.socket) + l = getline(conn.transport) + reply = parseline(l, conn.transport) convert_reply(reply) end parse_error(l::AbstractString) = throw(ServerException(l)) -function parse_bulk_string(s::TCPSocket, slen::Int) - b = read(s, slen+2) # add crlf +function parse_bulk_string(t::Transport.RedisTransport, slen::Int) + b = Transport.read_nbytes(t, slen+2) # add crlf if length(b) != slen + 2 throw(ProtocolException( "Bulk string read error: expected $slen bytes; received $(length(b))" @@ -30,17 +30,17 @@ function parse_bulk_string(s::TCPSocket, slen::Int) end end -function parse_array(s::TCPSocket, slen::Int) +function parse_array(t::Transport.RedisTransport, slen::Int) a = Array{Any, 1}(undef, slen) for i = 1:slen - l = getline(s) - r = parseline(l, s) + l = getline(t) + r = parseline(l, t) a[i] = r end return a end -function parseline(l::AbstractString, s::TCPSocket) +function parseline(l::AbstractString, t::Transport.RedisTransport) reply_type = l[1] reply_token = l[2:end] if reply_type == '+' @@ -52,14 +52,14 @@ function parseline(l::AbstractString, s::TCPSocket) if slen == -1 nothing else - parse_bulk_string(s, slen) + parse_bulk_string(t, slen) end elseif reply_type == '*' slen = parse(Int, reply_token) if slen == -1 nothing else - parse_array(s, slen) + parse_array(t, slen) end elseif reply_type == '-' parse_error(reply_token) @@ -90,8 +90,8 @@ function execute_command_without_reply(conn::RedisConnectionBase, command) is_connected(conn) || throw(ConnectionException("Socket is disconnected")) iob = IOBuffer() pack_command(iob, command) - lock(conn.socket.lock) do - write(conn.socket, take!(iob)) + Transport.io_lock(conn.transport) do + Transport.write_bytes(conn.transport, take!(iob)) end end diff --git a/src/transport/tcp.jl b/src/transport/tcp.jl new file mode 100644 index 0000000..baa4a85 --- /dev/null +++ b/src/transport/tcp.jl @@ -0,0 +1,19 @@ +struct TCPTransport <: RedisTransport + sock::TCPSocket +end + +read_line(t::TCPTransport) = readline(t.sock) +read_nbytes(t::TCPTransport, m::Int) = read(t.sock, m) +write_bytes(t::TCPTransport, b::Vector{UInt8}) = write(t.sock, b) +Base.close(t::TCPTransport) = close(t.sock) +function set_props!(t::TCPTransport) + # disable nagle and enable quickack to speed up the usually small exchanges + Sockets.nagle(t.sock, false) + Sockets.quickack(t.sock, true) +end +get_sslconfig(::TCPTransport) = nothing +io_lock(f, t::TCPTransport) = lock(f, t.sock.lock) +function is_connected(t::TCPTransport) + status = t.sock.status + status == StatusActive || status == StatusOpen || status == StatusPaused +end diff --git a/src/transport/tls.jl b/src/transport/tls.jl new file mode 100644 index 0000000..7656a99 --- /dev/null +++ b/src/transport/tls.jl @@ -0,0 +1,56 @@ +struct TLSTransport <: RedisTransport + sock::TCPSocket + ctx::MbedTLS.SSLContext + sslconfig::MbedTLS.SSLConfig + buff::IOBuffer + + function TLSTransport(sock::TCPSocket, sslconfig::MbedTLS.SSLConfig) + ctx = MbedTLS.SSLContext() + MbedTLS.setup!(ctx, sslconfig) + MbedTLS.associate!(ctx, sock) + MbedTLS.handshake(ctx) + + return new(sock, ctx, sslconfig, PipeBuffer()) + end +end + +function read_into_buffer_until(cond::Function, t::TLSTransport) + cond(t) && return + + buff = Vector{UInt8}(undef, MbedTLS.MBEDTLS_SSL_MAX_CONTENT_LEN) + pbuff = pointer(buff) + + while !cond(t) && !eof(t.ctx) + nread = readbytes!(t.ctx, buff; all=false) + if nread > 0 + unsafe_write(t.buff, pbuff, nread) + end + end +end + +function read_line(t::TLSTransport) + read_into_buffer_until(t) do t + iob = t.buff + (bytesavailable(t.buff) > 0) && (UInt8('\n') in view(iob.data, iob.ptr:iob.size)) + end + return readline(t.buff) +end +function read_nbytes(t::TLSTransport, m::Int) + read_into_buffer_until(t) do t + bytesavailable(t.buff) >= m + end + return read(t.buff, m) +end +write_bytes(t::TLSTransport, b::Vector{UInt8}) = write(t.ctx, b) +Base.close(t::TLSTransport) = close(t.ctx) +function set_props!(s::TLSTransport) + # disable nagle and enable quickack to speed up the usually small exchanges + Sockets.nagle(s.sock, false) + Sockets.quickack(s.sock, true) +end +get_sslconfig(t::TLSTransport) = t.sslconfig +io_lock(f, t::TLSTransport) = lock(f, t.sock.lock) +function is_connected(t::TLSTransport) + status = t.sock.status + status == StatusActive || status == StatusOpen || status == StatusPaused +end diff --git a/src/transport/transport.jl b/src/transport/transport.jl new file mode 100644 index 0000000..b8c8038 --- /dev/null +++ b/src/transport/transport.jl @@ -0,0 +1,33 @@ +""" + Transport module for Redis.jl abstractes the connection to the Redis server. + +Each transportimplementation must provide the following methods: +- `read_line(t::RedisTransport)`: read one line from the transport, similar to `readline` +- `read_nbytes(t::RedisTransport, m::Int)`: read `m` bytes from the transport, similar to `read` +- `write_bytes(t::RedisTransport, b::Vector{UInt8})`: write bytes to the transport, similar to `write` +- `close(t::RedisTransport)` +- `is_connected(t::RedisTransport)`: whether the transport is connected or not +- `status(t::RedisTransport)`: status of the transport, whether it is connected or not +- `set_props!(t::RedisTransport)`: set any properties required. For example, disable nagle and enable quickack to speed up the usually small exchanges +- `get_sslconfig(t::RedisTransport)`: get the SSL configuration for the transport if applicable +- `io_lock(f, t::RedisTransport)`: lock the transport for IO operations + +""" +module Transport + +using Sockets +using MbedTLS + +import Sockets.connect, Sockets.TCPSocket, Base.StatusActive, Base.StatusOpen, Base.StatusPaused + +abstract type RedisTransport end + +include("tls.jl") +include("tcp.jl") + +function transport(host::AbstractString, port::Integer, sslconfig::Union{MbedTLS.SSLConfig, Nothing}=nothing) + socket = connect(host, port) + return (sslconfig !== nothing) ? TLSTransport(socket, sslconfig) : TCPTransport(socket) +end + +end # module Transport \ No newline at end of file diff --git a/test/client_tests.jl b/test/client_tests.jl index 7354134..d55c084 100644 --- a/test/client_tests.jl +++ b/test/client_tests.jl @@ -1,30 +1,32 @@ import Redis: flatten, flatten_command, convert_response -@testset "Flatten" begin - @test flatten("simple") == "simple" - @test flatten(1) == "1" - @test flatten(2.5) == "2.5" - @test flatten([1, "2", 3.5]) == ["1", "2", "3.5"] +function client_tests() + @testset "Flatten" begin + @test flatten("simple") == "simple" + @test flatten(1) == "1" + @test flatten(2.5) == "2.5" + @test flatten([1, "2", 3.5]) == ["1", "2", "3.5"] - s = Set([1, 5, "10.9"]) - @test Set(flatten(s)) == Set(["1", "5", "10.9"]) + s = Set([1, 5, "10.9"]) + @test Set(flatten(s)) == Set(["1", "5", "10.9"]) - d = Dict{Any, Any}(1 => 2, 3 => 4) - @test Set(flatten(d)) == Set(["1", "2", "3", "4"]) -end + d = Dict{Any, Any}(1 => 2, 3 => 4) + @test Set(flatten(d)) == Set(["1", "2", "3", "4"]) + end -@testset "Commands" begin - result = flatten_command(1, 2, ["4", "5", 6.7], 8) - @test result == ["1", "2", "4", "5", "6.7", "8"] -end + @testset "Commands" begin + result = flatten_command(1, 2, ["4", "5", 6.7], 8) + @test result == ["1", "2", "4", "5", "6.7", "8"] + end -@testset "Convert" begin - @test convert_response(Dict{AbstractString, AbstractString}, ["1","2","3","4"]) == Dict("1" => "2", "3" => "4") - @test convert_response(Dict{AbstractString, AbstractString}, []) == Dict() - @test convert_response(Float64, "12.3") ≈ 12.3 - @test convert_response(Float64, 10) ≈ 10. - @test convert_response(Bool, "OK") - @test !convert_response(Bool, "f") - @test convert_response(Bool, 1) - @test !convert_response(Bool, 0) -end + @testset "Convert" begin + @test convert_response(Dict{AbstractString, AbstractString}, ["1","2","3","4"]) == Dict("1" => "2", "3" => "4") + @test convert_response(Dict{AbstractString, AbstractString}, []) == Dict() + @test convert_response(Float64, "12.3") ≈ 12.3 + @test convert_response(Float64, 10) ≈ 10. + @test convert_response(Bool, "OK") + @test !convert_response(Bool, "f") + @test convert_response(Bool, 1) + @test !convert_response(Bool, 0) + end +end \ No newline at end of file diff --git a/test/redis_tests.jl b/test/redis_tests.jl index 090a7f2..b0b577a 100644 --- a/test/redis_tests.jl +++ b/test/redis_tests.jl @@ -1,6 +1,7 @@ -conn = RedisConnection() -flushall(conn) +# constants for code legibility +const REDIS_PERSISTENT_KEY = -1 +const REDIS_EXPIRED_KEY = -2 # some random key names testkey = "Redis_Test_"*randstring() @@ -14,429 +15,432 @@ s1 = randstring(); s2 = randstring(); s3 = randstring() s4 = randstring(); s5 = randstring(); s6 = randstring() s7 = randstring(); s8 = randstring(); s9 = randstring() -# constants for code legibility -const REDIS_PERSISTENT_KEY = -1 -const REDIS_EXPIRED_KEY = -2 - -@testset "Strings" begin - @test set(conn, testkey, s1) - @test get(conn, testkey) == s1 - @test exists(conn, testkey) - @test keys(conn, testkey) == Set([testkey]) - @test del(conn, testkey, "notakey", "notakey2") == 1 # only 1 of 3 key exists - - # 'NIL' - @test get(conn, "notakey") === nothing - - set(conn, testkey, s1) - set(conn, testkey2, s2) - set(conn, testkey3, s3) - # RANDOMKEY can return 'NIL', so it returns Union{Nothing, T}. KEYS * always returns empty Set when Redis is empty - @test randomkey(conn) in keys(conn, "*") - @test getrange(conn, testkey, 0, 3) == s1[1:4] - - @test set(conn, testkey, 2) - @test incr(conn, testkey) == 3 - @test incrby(conn, testkey, 3) == 6 - @test incrbyfloat(conn, testkey, 1.5) == "7.5" - @test mget(conn, testkey, testkey2, testkey3) == ["7.5", s2, s3] - @test strlen(conn, testkey2) == length(s2) - @test rename(conn, testkey2, testkey4) == "OK" - @test testkey4 in keys(conn,"*") - del(conn, testkey, testkey2, testkey3, testkey4) - - @test append(conn, testkey, s1) == length(s1) - @test append(conn, testkey, s2) == length(s1) + length(s2) - get(conn, testkey) == string(s1, s2) - del(conn, testkey) -end - -@testset "Bits" begin - @test setbit(conn,testkey, 0, 1) == 0 - @test setbit(conn,testkey, 2, 1) == 0 - @test getbit(conn, testkey, 0) == 1 - @test getbit(conn, testkey, 1) == 0 # default is 0 - @test getbit(conn, testkey, 2) == 1 - @test bitcount(conn, testkey) == 2 - del(conn, testkey) - - for i=0:3 - setbit(conn, testkey, i, 1) - setbit(conn, testkey2, i, 1) +function redis_tests(conn = RedisConnection()) + flushall(conn) + + @testset "Strings" begin + @test set(conn, testkey, s1) + @test get(conn, testkey) == s1 + @test exists(conn, testkey) + @test keys(conn, testkey) == Set([testkey]) + @test del(conn, testkey, "notakey", "notakey2") == 1 # only 1 of 3 key exists + + # 'NIL' + @test get(conn, "notakey") === nothing + + set(conn, testkey, s1) + set(conn, testkey2, s2) + set(conn, testkey3, s3) + # RANDOMKEY can return 'NIL', so it returns Union{Nothing, T}. KEYS * always returns empty Set when Redis is empty + @test randomkey(conn) in keys(conn, "*") + @test getrange(conn, testkey, 0, 3) == s1[1:4] + + @test set(conn, testkey, 2) + @test incr(conn, testkey) == 3 + @test incrby(conn, testkey, 3) == 6 + @test incrbyfloat(conn, testkey, 1.5) == "7.5" + @test mget(conn, testkey, testkey2, testkey3) == ["7.5", s2, s3] + @test strlen(conn, testkey2) == length(s2) + @test rename(conn, testkey2, testkey4) == "OK" + @test testkey4 in keys(conn,"*") + del(conn, testkey, testkey2, testkey3, testkey4) + + @test append(conn, testkey, s1) == length(s1) + @test append(conn, testkey, s2) == length(s1) + length(s2) + get(conn, testkey) == string(s1, s2) + del(conn, testkey) end - @test bitop(conn, "AND", testkey3, testkey, testkey2) == 1 - for i=0:3 - setbit(conn, testkey, i, 1) - setbit(conn, testkey2, i, 0) + @testset "Bits" begin + @test setbit(conn,testkey, 0, 1) == 0 + @test setbit(conn,testkey, 2, 1) == 0 + @test getbit(conn, testkey, 0) == 1 + @test getbit(conn, testkey, 1) == 0 # default is 0 + @test getbit(conn, testkey, 2) == 1 + @test bitcount(conn, testkey) == 2 + del(conn, testkey) + + for i=0:3 + setbit(conn, testkey, i, 1) + setbit(conn, testkey2, i, 1) + end + @test bitop(conn, "AND", testkey3, testkey, testkey2) == 1 + + for i=0:3 + setbit(conn, testkey, i, 1) + setbit(conn, testkey2, i, 0) + end + bitop(conn, "AND", testkey3, testkey, testkey2) + @test [getbit(conn, testkey3, i) for i in 0:3] == zeros(4) + + @test bitop(conn, "OR", testkey3, testkey, testkey2) == 1 + @test [getbit(conn, testkey3, i) for i in 0:3] == ones(4) + + setbit(conn, testkey, 0, 0) + setbit(conn, testkey, 1, 0) + setbit(conn, testkey2, 1, 1) + setbit(conn, testkey2, 3, 1) + @test bitop(conn, "XOR", testkey3, testkey, testkey2) == 1 + @test [getbit(conn, testkey3, i) for i in 0:3] == [0; 1; 1; 0] + + @test bitop(conn, "NOT", testkey3, testkey3) == 1 + @test [getbit(conn, testkey3, i) for i in 0:3] == [1; 0; 0; 1] + del(conn, testkey, testkey2, testkey3) end - bitop(conn, "AND", testkey3, testkey, testkey2) - @test [getbit(conn, testkey3, i) for i in 0:3] == zeros(4) - - @test bitop(conn, "OR", testkey3, testkey, testkey2) == 1 - @test [getbit(conn, testkey3, i) for i in 0:3] == ones(4) - - setbit(conn, testkey, 0, 0) - setbit(conn, testkey, 1, 0) - setbit(conn, testkey2, 1, 1) - setbit(conn, testkey2, 3, 1) - @test bitop(conn, "XOR", testkey3, testkey, testkey2) == 1 - @test [getbit(conn, testkey3, i) for i in 0:3] == [0; 1; 1; 0] - - @test bitop(conn, "NOT", testkey3, testkey3) == 1 - @test [getbit(conn, testkey3, i) for i in 0:3] == [1; 0; 0; 1] - del(conn, testkey, testkey2, testkey3) -end -@testset "Dump" begin - # TODO: DUMP AND RESTORE HAVE ISSUES - #= - set(conn, testkey, "10") - # this line passes test when a client is available: - @test [UInt8(x) for x in Redis.dump(r, testkey)] == readbytes(`redis-cli dump t`)[1:end-1] - =# - - #= this causes 'ERR DUMP payload version or checksum are wrong', a TODO: need to - translate the return value and send it back correctly - set(conn, testkey, 1) - redisdump = Redis.dump(conn, testkey) - del(conn, testkey) - restore(conn, testkey, 0, redisdump) - =# -end + @testset "Dump" begin + # TODO: DUMP AND RESTORE HAVE ISSUES + #= + set(conn, testkey, "10") + # this line passes test when a client is available: + @test [UInt8(x) for x in Redis.dump(r, testkey)] == readbytes(`redis-cli dump t`)[1:end-1] + =# + + #= this causes 'ERR DUMP payload version or checksum are wrong', a TODO: need to + translate the return value and send it back correctly + set(conn, testkey, 1) + redisdump = Redis.dump(conn, testkey) + del(conn, testkey) + restore(conn, testkey, 0, redisdump) + =# + end -@testset "Migrate" begin - # TODO: test of `migrate` requires 2 server instances in Travis - set(conn, testkey, s1) - @test move(conn, testkey, 1) - @test exists(conn, testkey) == false - @test Redis.select(conn, 1) == "OK" - @test get(conn, testkey) == s1 - del(conn, testkey) - Redis.select(conn, 0) -end + @testset "Migrate" begin + # TODO: test of `migrate` requires 2 server instances in Travis + set(conn, testkey, s1) + @test move(conn, testkey, 1) + @test exists(conn, testkey) == false + @test Redis.select(conn, 1) == "OK" + @test get(conn, testkey) == s1 + del(conn, testkey) + Redis.select(conn, 0) + end -@testset "Expiry" begin - set(conn, testkey, s1) - expire(conn, testkey, 1) - sleep(1) - @test exists(conn, testkey) == false - - set(conn, testkey, s1) - expireat(conn, testkey, round(Int, Dates.datetime2unix(time(conn)+Dates.Second(1)))) - sleep(2) # only passes test with 2 second delay - @test exists(conn, testkey) == false - - set(conn, testkey, s1) - @test pexpire(conn, testkey, 1) - @test ttl(conn, testkey) == REDIS_EXPIRED_KEY - - set(conn, testkey, s1) - @test pexpire(conn, testkey, 2000) - @test pttl(conn, testkey) > 100 - @test persist(conn, testkey) - @test ttl(conn, testkey) == REDIS_PERSISTENT_KEY - del(conn, testkey, testkey2, testkey3) -end + @testset "Expiry" begin + set(conn, testkey, s1) + expire(conn, testkey, 1) + sleep(1) + @test exists(conn, testkey) == false + + set(conn, testkey, s1) + expireat(conn, testkey, round(Int, Dates.datetime2unix(time(conn)+Dates.Second(1)))) + sleep(2) # only passes test with 2 second delay + @test exists(conn, testkey) == false + + set(conn, testkey, s1) + @test pexpire(conn, testkey, 1) + sleep(1) # ensure key has expired + @test ttl(conn, testkey) == REDIS_EXPIRED_KEY + + set(conn, testkey, s1) + @test pexpire(conn, testkey, 2000) + @test pttl(conn, testkey) > 100 + @test persist(conn, testkey) + @test ttl(conn, testkey) == REDIS_PERSISTENT_KEY + del(conn, testkey, testkey2, testkey3) + end -@testset "Lists" begin - @test lpush(conn, testkey, s1, s2, "a", "a", s3, s4) == 6 - @test lpop(conn, testkey) == s4 - @test rpop(conn, testkey) == s1 - @test lpop(conn, "non_existent_list") === nothing - @test rpop(conn, "non_existent_list") === nothing - @test llen(conn, testkey) == 4 - @test lindex(conn, "non_existent_list", 1) === nothing - @test lindex(conn, testkey, 0) == s3 - @test lindex(conn, testkey, 10) === nothing - @test lrem(conn, testkey, 0, "a") == 2 - @test lset(conn, testkey, 0, s5) == "OK" - @test lindex(conn, testkey, 0) == s5 - @test linsert(conn, testkey, "BEFORE", s2, s3) == 3 - @test linsert(conn, testkey, "AFTER", s3, s6) == 4 - @test lpushx(conn, testkey2, "nothing") == false - @test rpushx(conn, testkey2, "nothing") == false - @test ltrim(conn, testkey, 0, 1) == "OK" - @test lrange(conn, testkey, 0, -1) == [s5; s3] - @test brpop(conn, testkey, 0) == [testkey, s3] - lpush(conn, testkey, s3) - @test blpop(conn, testkey, 0) == [testkey, s3] - lpush(conn, testkey, s4) - lpush(conn, testkey, s3) - listvals = [s3; s4; s5] - for i in 1:3 - @test rpoplpush(conn, testkey, testkey2) == listvals[4-i] # rpop + @testset "Lists" begin + @test lpush(conn, testkey, s1, s2, "a", "a", s3, s4) == 6 + @test lpop(conn, testkey) == s4 + @test rpop(conn, testkey) == s1 + @test lpop(conn, "non_existent_list") === nothing + @test rpop(conn, "non_existent_list") === nothing + @test llen(conn, testkey) == 4 + @test lindex(conn, "non_existent_list", 1) === nothing + @test lindex(conn, testkey, 0) == s3 + @test lindex(conn, testkey, 10) === nothing + @test lrem(conn, testkey, 0, "a") == 2 + @test lset(conn, testkey, 0, s5) == "OK" + @test lindex(conn, testkey, 0) == s5 + @test linsert(conn, testkey, "BEFORE", s2, s3) == 3 + @test linsert(conn, testkey, "AFTER", s3, s6) == 4 + @test lpushx(conn, testkey2, "nothing") == false + @test rpushx(conn, testkey2, "nothing") == false + @test ltrim(conn, testkey, 0, 1) == "OK" + @test lrange(conn, testkey, 0, -1) == [s5; s3] + @test brpop(conn, testkey, 0) == [testkey, s3] + lpush(conn, testkey, s3) + @test blpop(conn, testkey, 0) == [testkey, s3] + lpush(conn, testkey, s4) + lpush(conn, testkey, s3) + listvals = [s3; s4; s5] + for i in 1:3 + @test rpoplpush(conn, testkey, testkey2) == listvals[4-i] # rpop + end + @test rpoplpush(conn, testkey, testkey2) === nothing + @test llen(conn, testkey) == 0 + @test llen(conn, testkey2) == 3 + @test lrange(conn, testkey2, 0, -1) == listvals + for i in 1:3 + @test brpoplpush(conn, testkey2, testkey, 0) == listvals[4-i] # rpop + end + @test lrange(conn, testkey, 0, -1) == listvals + + # the following command can only be applied to lists containing numeric values + sortablelist = [pi, 1, 2] + lpush(conn, testkey3, sortablelist) + @test Redis.sort(conn, testkey3) == ["1.0", "2.0", "3.141592653589793"] + del(conn, testkey, testkey2, testkey3) end - @test rpoplpush(conn, testkey, testkey2) === nothing - @test llen(conn, testkey) == 0 - @test llen(conn, testkey2) == 3 - @test lrange(conn, testkey2, 0, -1) == listvals - for i in 1:3 - @test brpoplpush(conn, testkey2, testkey, 0) == listvals[4-i] # rpop + + @testset "Hashes" begin + @test hmset(conn, testhash, Dict(1 => 2, "3" => 4, "5" => "6")) + @test hscan(conn, testhash, 0) == (0, Dict("1" => "2", "3" => "4", "5" => "6")) + @test hexists(conn, testhash, 1) == true + @test hexists(conn, testhash, "1") == true + @test hget(conn, testhash, 1) == "2" + @test hgetall(conn, testhash) == Dict("1" => "2", "3" => "4", "5" => "6") + + @test hget(conn, testhash, "non_existent_field") === nothing + @test hmget(conn, testhash, 1, 3) == ["2", "4"] + a = hmget(conn, testhash, "non_existent_field1", "non_existent_field2") + @test a[1] === nothing + @test a[2] === nothing + + @test Set(hvals(conn, testhash)) == Set(["2", "4", "6"]) # use Set for comp as hash ordering is random + @test Set(hkeys(conn, testhash)) == Set(["1", "3", "5"]) + @test hset(conn, testhash, "3", 10) == false # if the field already hset returns false + @test hget(conn, testhash, "3") == "10" # but still sets it to the new value + @test hset(conn, testhash, "10", "10") == true # new field hset returns true + @test hget(conn, testhash, "10") == "10" # correctly set new field + @test hsetnx(conn, testhash, "1", "10") == false # field exists + @test hsetnx(conn, testhash, "11", "10") == true # field doesn't exist + @test hlen(conn, testhash) == 5 # testhash now has 5 fields + @test hincrby(conn, testhash, "1", 1) == 3 + @test parse(Float64, hincrbyfloat(conn, testhash, "1", 1.5)) == 4.5 + + del(conn, testhash) + + N = 1500 + @test hmset(conn, testhash, Dict(i=>i for i=1:N)) + (cursor, d) = hscan(conn, testhash, 0) + while cursor != 0 + (cursor, d_new) = hscan(conn, testhash, cursor) + merge!(d, d_new) + end + @test d == Dict(string(i)=>string(i) for i=1:N) + del(conn, testhash) end - @test lrange(conn, testkey, 0, -1) == listvals - # the following command can only be applied to lists containing numeric values - sortablelist = [pi, 1, 2] - lpush(conn, testkey3, sortablelist) - @test Redis.sort(conn, testkey3) == ["1.0", "2.0", "3.141592653589793"] - del(conn, testkey, testkey2, testkey3) -end + @testset "Sets" begin + @test sadd(conn, testkey, s1) == true + @test sadd(conn, testkey, s1) == false # already exists + @test sadd(conn, testkey, s2) == true + @test smembers(conn, testkey) == Set([s1, s2]) + @test scard(conn, testkey) == 2 + sadd(conn, testkey, s3) + @test smove(conn, testkey, testkey2, s3) == true + @test sismember(conn, testkey2, s3) == true + sadd(conn, testkey2, s2) + @test sunion(conn, testkey, testkey2) == Set([s1, s2, s3]) + @test sunionstore(conn, testkey3, testkey, testkey2) == 3 + @test srem(conn, testkey3, s1, s2, s3) == 3 + @test smembers(conn, testkey3) == Set([]) + @test sinterstore(conn, testkey3, testkey, testkey2) == 1 + # only the following method returns 'nil' if the Set does not exist + @test srandmember(conn, testkey3) in Set([s1, s2, s3]) + @test srandmember(conn, "empty_set") === nothing + # this method returns an emtpty Set if the the Set is empty + @test issubset(srandmember(conn, testkey2, 2), Set([s1, s2, s3])) + @test srandmember(conn, "non_existent_set", 10) == Set{AbstractString}() + @test sdiff(conn, testkey, testkey2) == Set([s1]) + @test spop(conn, testkey) in Set([s1, s2, s3]) + @test spop(conn, "empty_set") === nothing + del(conn, testkey, testkey2, testkey3) + end -@testset "Hashes" begin - @test hmset(conn, testhash, Dict(1 => 2, "3" => 4, "5" => "6")) - @test hscan(conn, testhash, 0) == (0, Dict("1" => "2", "3" => "4", "5" => "6")) - @test hexists(conn, testhash, 1) == true - @test hexists(conn, testhash, "1") == true - @test hget(conn, testhash, 1) == "2" - @test hgetall(conn, testhash) == Dict("1" => "2", "3" => "4", "5" => "6") - - @test hget(conn, testhash, "non_existent_field") === nothing - @test hmget(conn, testhash, 1, 3) == ["2", "4"] - a = hmget(conn, testhash, "non_existent_field1", "non_existent_field2") - @test a[1] === nothing - @test a[2] === nothing - - @test Set(hvals(conn, testhash)) == Set(["2", "4", "6"]) # use Set for comp as hash ordering is random - @test Set(hkeys(conn, testhash)) == Set(["1", "3", "5"]) - @test hset(conn, testhash, "3", 10) == false # if the field already hset returns false - @test hget(conn, testhash, "3") == "10" # but still sets it to the new value - @test hset(conn, testhash, "10", "10") == true # new field hset returns true - @test hget(conn, testhash, "10") == "10" # correctly set new field - @test hsetnx(conn, testhash, "1", "10") == false # field exists - @test hsetnx(conn, testhash, "11", "10") == true # field doesn't exist - @test hlen(conn, testhash) == 5 # testhash now has 5 fields - @test hincrby(conn, testhash, "1", 1) == 3 - @test parse(Float64, hincrbyfloat(conn, testhash, "1", 1.5)) == 4.5 - - del(conn, testhash) - - N = 1500 - @test hmset(conn, testhash, Dict(i=>i for i=1:N)) - (cursor, d) = hscan(conn, testhash, 0) - while cursor != 0 - (cursor, d_new) = hscan(conn, testhash, cursor) - merge!(d, d_new) + @testset "Sorted Sets" begin + @test zadd(conn, testkey, 0, s1) == true + @test zadd(conn, testkey, 1., s1) == false + @test zadd(conn, testkey, 1., s2) == true + @test zrange(conn, testkey, 0, -1) == OrderedSet([s1, s2]) + @test zcard(conn, testkey) == 2 + zadd(conn, testkey, 1.5, s3) + @test zcount(conn, testkey, 0, 1) == 2 # range as int + @test zcount(conn, testkey, "-inf", "+inf") == 3 # range as string + @test zincrby(conn, testkey, 1, s1) == "2" + @test parse(Float64, zincrby(conn, testkey, 1.2, s1)) == 3.2 + @test zrem(conn, testkey, s1, s2) == 2 + del(conn, testkey) + + @test zadd(conn, testkey, zip(zeros(3), [s1, s2, s3])...) == 3 + del(conn, testkey) + + vals = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"] + + # tests where all scores == 0 + zadd(conn, testkey, zip(zeros(length(vals)), vals)...) + @test zlexcount(conn, testkey, "-", "+") == length(vals) + @test zlexcount(conn, testkey, "[b", "[f") == 5 + @test zrangebylex(conn, testkey, "-", "[c") == OrderedSet(["a", "b", "c"]) + @test zrangebylex(conn, testkey, "[aa", "(g") == OrderedSet(["b", "c", "d", "e", "f"]) + @test zrangebylex(conn, testkey, "[a", "(g") == OrderedSet(["a", "b", "c", "d", "e", "f"]) + @test zremrangebylex(conn, testkey, "[a", "[h") == 8 + @test zrange(conn, testkey, 0, -1) == OrderedSet(["i", "j"]) + del(conn, testkey) + + # tests where scores are sequence 1:10 + zadd(conn, testkey, zip(1:length(vals), vals)...) + @test zrangebyscore(conn, testkey, "(1", "2") == OrderedSet(["b"]) + @test zrangebyscore(conn, testkey, "1", "2") == OrderedSet(["a", "b"]) + @test zrangebyscore(conn, testkey, "(1", "(2") == OrderedSet([]) + @test zrank(conn, testkey, "d") == 3 # redis arrays 0-base + + # 'NIL' + @test zrank(conn, testkey, "z") === nothing + del(conn, testkey) + + zadd(conn, testkey, zip(1:length(vals), vals)...) + @test zremrangebyrank(conn, testkey, 0, 1) == 2 + @test zrange(conn, testkey, 0, -1, "WITHSCORES") == OrderedSet(["c", "3", "d", "4", "e", "5", "f", "6", "g", "7", "h", "8", "i", "9", "j", "10"]) + @test zremrangebyscore(conn, testkey, "-inf", "(5") == 2 + @test zrange(conn, testkey, 0, -1, "WITHSCORES") == OrderedSet(["e", "5", "f", "6", "g", "7", "h", "8", "i", "9", "j", "10"]) + @test zrevrange(conn, testkey, 0, -1) == OrderedSet(["j", "i", "h", "g", "f", "e"]) + @test zrevrangebyscore(conn, testkey, "+inf", "-inf") == OrderedSet(["j", "i", "h", "g", "f", "e"]) + @test zrevrangebyscore(conn, testkey, "+inf", "-inf", "WITHSCORES", "LIMIT", 2, 3) == OrderedSet(["h", "8", "g", "7", "f", "6"]) + @test zrevrangebyscore(conn, testkey, 7, 5) == OrderedSet(["g", "f", "e"]) + @test zrevrangebyscore(conn, testkey, "(6", "(5") == OrderedSet{AbstractString}() + @test zrevrank(conn, testkey, "e") == 5 + @test zrevrank(conn, "ordered_set", "non_existent_member") === nothing + @test zscore(conn, testkey, "e") == "5" + @test zscore(conn, "ordered_set", "non_existent_member") === nothing + del(conn, testkey) + + vals2 = ["a", "b", "c", "d"] + zadd(conn, testkey, zip(1:length(vals), vals)...) + zadd(conn, testkey2, zip(1:length(vals2), vals2)...) + @test zunionstore(conn, testkey3, 2, [testkey, testkey2]) == 10 + @test zrange(conn, testkey3, 0, -1) == OrderedSet(vals) + del(conn, testkey3) + + zunionstore(conn, testkey3, 2, [testkey, testkey2], [2; 3]) + @test zrange(conn, testkey3, 0, -1) == OrderedSet(["a", "b", "e", "f", "g", "c", "h", "i", "d", "j"]) + zunionstore(conn, testkey3, 2, [testkey, testkey2], [2; 3], aggregate=Aggregate.Max) + @test zrange(conn, testkey3, 0, -1) == OrderedSet(["a", "b", "c", "e", "d", "f", "g", "h", "i", "j"]) + zunionstore(conn, testkey3, 2, [testkey, testkey2], [2; 3], aggregate=Aggregate.Min) + @test zrange(conn, testkey3, 0, -1) == OrderedSet(["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]) + del(conn, testkey3) + + vals2 = ["a", "b", "c", "d"] + @test zinterstore(conn, testkey3, 2, [testkey, testkey2]) == 4 + del(conn, testkey, testkey2, testkey3) end - @test d == Dict(string(i)=>string(i) for i=1:N) - del(conn, testhash) -end -@testset "Sets" begin - @test sadd(conn, testkey, s1) == true - @test sadd(conn, testkey, s1) == false # already exists - @test sadd(conn, testkey, s2) == true - @test smembers(conn, testkey) == Set([s1, s2]) - @test scard(conn, testkey) == 2 - sadd(conn, testkey, s3) - @test smove(conn, testkey, testkey2, s3) == true - @test sismember(conn, testkey2, s3) == true - sadd(conn, testkey2, s2) - @test sunion(conn, testkey, testkey2) == Set([s1, s2, s3]) - @test sunionstore(conn, testkey3, testkey, testkey2) == 3 - @test srem(conn, testkey3, s1, s2, s3) == 3 - @test smembers(conn, testkey3) == Set([]) - @test sinterstore(conn, testkey3, testkey, testkey2) == 1 - # only the following method returns 'nil' if the Set does not exist - @test srandmember(conn, testkey3) in Set([s1, s2, s3]) - @test srandmember(conn, "empty_set") === nothing - # this method returns an emtpty Set if the the Set is empty - @test issubset(srandmember(conn, testkey2, 2), Set([s1, s2, s3])) - @test srandmember(conn, "non_existent_set", 10) == Set{AbstractString}() - @test sdiff(conn, testkey, testkey2) == Set([s1]) - @test spop(conn, testkey) in Set([s1, s2, s3]) - @test spop(conn, "empty_set") === nothing - del(conn, testkey, testkey2, testkey3) -end + @testset "Scan" begin -@testset "Sorted Sets" begin - @test zadd(conn, testkey, 0, s1) == true - @test zadd(conn, testkey, 1., s1) == false - @test zadd(conn, testkey, 1., s2) == true - @test zrange(conn, testkey, 0, -1) == OrderedSet([s1, s2]) - @test zcard(conn, testkey) == 2 - zadd(conn, testkey, 1.5, s3) - @test zcount(conn, testkey, 0, 1) == 2 # range as int - @test zcount(conn, testkey, "-inf", "+inf") == 3 # range as string - @test zincrby(conn, testkey, 1, s1) == "2" - @test parse(Float64, zincrby(conn, testkey, 1.2, s1)) == 3.2 - @test zrem(conn, testkey, s1, s2) == 2 - del(conn, testkey) - - @test zadd(conn, testkey, zip(zeros(3), [s1, s2, s3])...) == 3 - del(conn, testkey) - - vals = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"] - - # tests where all scores == 0 - zadd(conn, testkey, zip(zeros(length(vals)), vals)...) - @test zlexcount(conn, testkey, "-", "+") == length(vals) - @test zlexcount(conn, testkey, "[b", "[f") == 5 - @test zrangebylex(conn, testkey, "-", "[c") == OrderedSet(["a", "b", "c"]) - @test zrangebylex(conn, testkey, "[aa", "(g") == OrderedSet(["b", "c", "d", "e", "f"]) - @test zrangebylex(conn, testkey, "[a", "(g") == OrderedSet(["a", "b", "c", "d", "e", "f"]) - @test zremrangebylex(conn, testkey, "[a", "[h") == 8 - @test zrange(conn, testkey, 0, -1) == OrderedSet(["i", "j"]) - del(conn, testkey) - - # tests where scores are sequence 1:10 - zadd(conn, testkey, zip(1:length(vals), vals)...) - @test zrangebyscore(conn, testkey, "(1", "2") == OrderedSet(["b"]) - @test zrangebyscore(conn, testkey, "1", "2") == OrderedSet(["a", "b"]) - @test zrangebyscore(conn, testkey, "(1", "(2") == OrderedSet([]) - @test zrank(conn, testkey, "d") == 3 # redis arrays 0-base - - # 'NIL' - @test zrank(conn, testkey, "z") === nothing - del(conn, testkey) - - zadd(conn, testkey, zip(1:length(vals), vals)...) - @test zremrangebyrank(conn, testkey, 0, 1) == 2 - @test zrange(conn, testkey, 0, -1, "WITHSCORES") == OrderedSet(["c", "3", "d", "4", "e", "5", "f", "6", "g", "7", "h", "8", "i", "9", "j", "10"]) - @test zremrangebyscore(conn, testkey, "-inf", "(5") == 2 - @test zrange(conn, testkey, 0, -1, "WITHSCORES") == OrderedSet(["e", "5", "f", "6", "g", "7", "h", "8", "i", "9", "j", "10"]) - @test zrevrange(conn, testkey, 0, -1) == OrderedSet(["j", "i", "h", "g", "f", "e"]) - @test zrevrangebyscore(conn, testkey, "+inf", "-inf") == OrderedSet(["j", "i", "h", "g", "f", "e"]) - @test zrevrangebyscore(conn, testkey, "+inf", "-inf", "WITHSCORES", "LIMIT", 2, 3) == OrderedSet(["h", "8", "g", "7", "f", "6"]) - @test zrevrangebyscore(conn, testkey, 7, 5) == OrderedSet(["g", "f", "e"]) - @test zrevrangebyscore(conn, testkey, "(6", "(5") == OrderedSet{AbstractString}() - @test zrevrank(conn, testkey, "e") == 5 - @test zrevrank(conn, "ordered_set", "non_existent_member") === nothing - @test zscore(conn, testkey, "e") == "5" - @test zscore(conn, "ordered_set", "non_existent_member") === nothing - del(conn, testkey) - - vals2 = ["a", "b", "c", "d"] - zadd(conn, testkey, zip(1:length(vals), vals)...) - zadd(conn, testkey2, zip(1:length(vals2), vals2)...) - @test zunionstore(conn, testkey3, 2, [testkey, testkey2]) == 10 - @test zrange(conn, testkey3, 0, -1) == OrderedSet(vals) - del(conn, testkey3) - - zunionstore(conn, testkey3, 2, [testkey, testkey2], [2; 3]) - @test zrange(conn, testkey3, 0, -1) == OrderedSet(["a", "b", "e", "f", "g", "c", "h", "i", "d", "j"]) - zunionstore(conn, testkey3, 2, [testkey, testkey2], [2; 3], aggregate=Aggregate.Max) - @test zrange(conn, testkey3, 0, -1) == OrderedSet(["a", "b", "c", "e", "d", "f", "g", "h", "i", "j"]) - zunionstore(conn, testkey3, 2, [testkey, testkey2], [2; 3], aggregate=Aggregate.Min) - @test zrange(conn, testkey3, 0, -1) == OrderedSet(["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]) - del(conn, testkey3) - - vals2 = ["a", "b", "c", "d"] - @test zinterstore(conn, testkey3, 2, [testkey, testkey2]) == 4 - del(conn, testkey, testkey2, testkey3) -end + end -@testset "Scan" begin + @testset "Scripting" begin + script = "return {KEYS[1], KEYS[2], ARGV[1], ARGV[2]}" + args = ["key1", "key2", "first", "second"] + resp = evalscript(conn, script, 2, args) + @test resp == args + del(conn, "key1") + + script = "return redis.call('set', KEYS[1], 'bar')" + ky = "foo" + resp = evalscript(conn, script, 1, [ky]) + @test resp == "OK" + del(conn, ky) + + + #@test evalscript(conn, "return {'1','2',{'3','Hello World!'}}", 0, []) == ["1"; "2"; ["3","Hello World!"]] + + # NOTE the truncated float, and truncated array in the response + # as per http://redis.io/commands/eval + # Lua has a single numerical type, Lua numbers. There is + # no distinction between integers and floats. So we always + # convert Lua numbers into integer replies, removing the + # decimal part of the number if any. If you want to return + # a float from Lua you should return it as a string, exactly + # like Redis itself does (see for instance the ZSCORE command). + # + # There is no simple way to have nils inside Lua arrays, + # this is a result of Lua table semantics, so when Redis + # converts a Lua array into Redis protocol the conversion + # is stopped if a nil is encountered. + #@test evalscript(conn, "return {1, 2, 3.3333, 'foo', nil, 'bar'}", 0, []) == [1, 2, 3, "foo"] + end -end + @testset "Transactions" begin + trans = open_transaction(conn) + @test set(trans, testkey, "foobar") == "QUEUED" + @test get(trans, testkey) == "QUEUED" + @test exec(trans) == ["OK", "foobar"] + @test del(trans, testkey) == "QUEUED" + @test exec(trans) == [true] + disconnect(trans) + end -@testset "Scripting" begin - script = "return {KEYS[1], KEYS[2], ARGV[1], ARGV[2]}" - args = ["key1", "key2", "first", "second"] - resp = evalscript(conn, script, 2, args) - @test resp == args - del(conn, "key1") - - script = "return redis.call('set', KEYS[1], 'bar')" - ky = "foo" - resp = evalscript(conn, script, 1, [ky]) - @test resp == "OK" - del(conn, ky) - - -#@test evalscript(conn, "return {'1','2',{'3','Hello World!'}}", 0, []) == ["1"; "2"; ["3","Hello World!"]] - -# NOTE the truncated float, and truncated array in the response -# as per http://redis.io/commands/eval -# Lua has a single numerical type, Lua numbers. There is -# no distinction between integers and floats. So we always -# convert Lua numbers into integer replies, removing the -# decimal part of the number if any. If you want to return -# a float from Lua you should return it as a string, exactly -# like Redis itself does (see for instance the ZSCORE command). -# -# There is no simple way to have nils inside Lua arrays, -# this is a result of Lua table semantics, so when Redis -# converts a Lua array into Redis protocol the conversion -# is stopped if a nil is encountered. -#@test evalscript(conn, "return {1, 2, 3.3333, 'foo', nil, 'bar'}", 0, []) == [1, 2, 3, "foo"] -end + @testset "Pipelines" begin + pipe = open_pipeline(conn) + set(pipe, testkey3, "anything") + @test length(read_pipeline(pipe)) == 1 + get(pipe, testkey3) + set(pipe, testkey4, "testing") + result = read_pipeline(pipe) + @test length(result) == 2 + @test result == ["anything", "OK"] + @test del(pipe, testkey3) == 1 + @test del(pipe, testkey4) == 2 + @test result == ["anything", "OK"] + disconnect(pipe) + end -@testset "Transactions" begin - trans = open_transaction(conn) - @test set(trans, testkey, "foobar") == "QUEUED" - @test get(trans, testkey) == "QUEUED" - @test exec(trans) == ["OK", "foobar"] - @test del(trans, testkey) == "QUEUED" - @test exec(trans) == [true] - disconnect(trans) -end + @testset "Pub/Sub" begin + function handleException(ex) + io = IOBuffer() + showerror(io, ex, catch_backtrace()) + err = String(take!(io)) + @warn "Error while processing subscription: $err" + return nothing + end -@testset "Pipelines" begin - pipe = open_pipeline(conn) - set(pipe, testkey3, "anything") - @test length(read_pipeline(pipe)) == 1 - get(pipe, testkey3) - set(pipe, testkey4, "testing") - result = read_pipeline(pipe) - @test length(result) == 2 - @test result == ["anything", "OK"] - @test del(pipe, testkey3) == 1 - @test del(pipe, testkey4) == 2 - @test result == ["anything", "OK"] - disconnect(pipe) -end + string_matched_results = Any[] + pattern_matched_results = Any[] -@testset "Pub/Sub" begin - function handleException(ex) - io = IOBuffer() - showerror(io, ex, catch_backtrace()) - err = String(take!(io)) - @warn "Error while processing subscription: $err" - return nothing - end + function string_matched(y::AbstractString) + push!(string_matched_results, y) + end - string_matched_results = Any[] - pattern_matched_results = Any[] + function pattern_matched(y::AbstractString) + push!(pattern_matched_results, y) + end - function string_matched(y::AbstractString) - push!(string_matched_results, y) - end + subs = open_subscription(conn, handleException) #handleException is called when an exception occurs - function pattern_matched(y::AbstractString) - push!(pattern_matched_results, y) - end + subscribe_data(subs, "1channel", string_matched) + psubscribe_data(subs, "1cha??el", pattern_matched) - subs = open_subscription(conn, handleException) #handleException is called when an exception occurs + subscribe(subs, "2channel", y->string_matched(y.message)) + psubscribe(subs, "2chan*", y->pattern_matched(y.message)) - subscribe_data(subs, "1channel", string_matched) - psubscribe_data(subs, "1cha??el", pattern_matched) + subscribe_data(subs, Dict("3channel" => string_matched)) + psubscribe_data(subs, Dict("3cha??el" => pattern_matched)) - subscribe(subs, "2channel", y->string_matched(y.message)) - psubscribe(subs, "2chan*", y->pattern_matched(y.message)) + subscribe(subs, Dict("4channel" => y->string_matched(y.message))) + psubscribe(subs, Dict("4chan*" => y->pattern_matched(y.message))) - subscribe_data(subs, Dict("3channel" => string_matched)) - psubscribe_data(subs, Dict("3cha??el" => pattern_matched)) + published_messages = ["hello, world1!", "hello, world 2!", "hello, world 3!", "hello, world 4!"] - subscribe(subs, Dict("4channel" => y->string_matched(y.message))) - psubscribe(subs, Dict("4chan*" => y->pattern_matched(y.message))) + sleep(10) # to ensure all subscriptions are registered + + for idx in 1:4 + @test publish(conn, string(idx)*"channel", published_messages[idx]) > 0 #Number of connected clients returned + end - published_messages = ["hello, world1!", "hello, world 2!", "hello, world 3!", "hello, world 4!"] + timedwait(5.0; pollint=1.0) do # wait for the messages to be received + length(string_matched_results) == 4 && length(pattern_matched_results) == 4 + end - for idx in 1:4 - @test publish(conn, string(idx)*"channel", published_messages[idx]) > 0 #Number of connected clients returned - end + @test string_matched_results == published_messages + @test pattern_matched_results == published_messages - timedwait(5.0; pollint=1.0) do # wait for the messages to be received - length(string_matched_results) == 4 && length(pattern_matched_results) == 4 + # following command prints ("Invalid response received: ") + disconnect(subs) + @info "This error is expected - to be addressed once the library is stable." end - @test string_matched_results == published_messages - @test pattern_matched_results == published_messages - - # following command prints ("Invalid response received: ") - disconnect(subs) - @info "This error is expected - to be addressed once the library is stable." + disconnect(conn) end - -disconnect(conn) diff --git a/test/runtests.jl b/test/runtests.jl index 28907df..3408b66 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,17 @@ using Random using Dates using Test using Base +using MbedTLS -include(joinpath(dirname(@__FILE__),"client_tests.jl")) -include(joinpath(dirname(@__FILE__),"redis_tests.jl")) +include("test_utils.jl") +include("client_tests.jl") +include("redis_tests.jl") + +client_tests() + +# TCP connection +redis_tests(RedisConnection()) + +# TLS connection +# TODO: enable after updating github CI configuration +# redis_tests(RedisConnection(; port=16379, sslconfig=client_tls_config("certs/ca.crt"))) \ No newline at end of file diff --git a/test/test_utils.jl b/test/test_utils.jl new file mode 100644 index 0000000..feb6cec --- /dev/null +++ b/test/test_utils.jl @@ -0,0 +1,32 @@ +function client_tls_config(cacrtfile::AbstractString, + clientcrtfile::Union{Nothing,AbstractString}=nothing, + clientkeyfile::Union{Nothing,AbstractString}=nothing, + verify::Bool=true +) + cacrt = MbedTLS.crt_parse_file(cacrtfile) + clientcrt = (clientcrtfile === nothing) ? nothing : MbedTLS.crt_parse_file(clientcrtfile) + clientkey = (clientkeyfile === nothing) ? nothing : MbedTLS.parse_keyfile(clientkeyfile) + client_tls_config(cacrt, clientcrt, clientkey, verify) +end + +function client_tls_config(cacrt::Union{Nothing,MbedTLS.CRT}=nothing, + clientcrt::Union{Nothing,MbedTLS.CRT}=nothing, + clientkey::Union{Nothing,MbedTLS.PKContext}=nothing, + verify::Bool=true +) + + conf = MbedTLS.SSLConfig() + MbedTLS.config_defaults!(conf) + + entropy = MbedTLS.Entropy() + rng = MbedTLS.CtrDrbg() + MbedTLS.seed!(rng, entropy) + MbedTLS.rng!(conf, rng) + + MbedTLS.authmode!(conf, verify ? MbedTLS.MBEDTLS_SSL_VERIFY_REQUIRED : MbedTLS.MBEDTLS_SSL_VERIFY_NONE) + + (cacrt === nothing) || MbedTLS.ca_chain!(conf, cacrt) + (clientcrt === nothing) || (clientkey === nothing) || MbedTLS.own_cert!(conf, clientcrt, clientkey) + + return conf +end From fbaeb90b094514f4328e99698b92dfc3211e83ed Mon Sep 17 00:00:00 2001 From: tan Date: Thu, 28 Dec 2023 12:47:33 +0530 Subject: [PATCH 2/5] include the continue-on-error flag for codecov --- .github/workflows/CI.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8d7b56f..2f83407 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -44,6 +44,7 @@ jobs: - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v2 + continue-on-error: true with: files: lcov.info fail_ci_if_error: false From ef6fcc79cab42486d0cfa387bca950f9dc8c6679 Mon Sep 17 00:00:00 2001 From: tan Date: Thu, 28 Dec 2023 14:48:41 +0530 Subject: [PATCH 3/5] start redis server with TLS --- .github/workflows/CI.yml | 15 +++++++----- test/certs/ca.crt | 24 +++++++++++++++++++ test/certs/ca.key | 28 ++++++++++++++++++++++ test/certs/certgen.sh | 18 +++++++++++++++ test/certs/server.crt | 22 ++++++++++++++++++ test/certs/server.csr | 18 +++++++++++++++ test/certs/server.key | 28 ++++++++++++++++++++++ test/certs/server.pem | 50 ++++++++++++++++++++++++++++++++++++++++ test/conf/redis.conf | 6 +++++ test/conf/redis.sh | 13 +++++++++++ 10 files changed, 216 insertions(+), 6 deletions(-) create mode 100644 test/certs/ca.crt create mode 100644 test/certs/ca.key create mode 100755 test/certs/certgen.sh create mode 100644 test/certs/server.crt create mode 100644 test/certs/server.csr create mode 100644 test/certs/server.key create mode 100644 test/certs/server.pem create mode 100644 test/conf/redis.conf create mode 100755 test/conf/redis.sh diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2f83407..9c2368f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -8,11 +8,6 @@ jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ubuntu-latest - services: - redis: - image: redis:7.2.3-bookworm # https://hub.docker.com/_/redis - ports: - - 6379:6379 strategy: fail-fast: false matrix: @@ -41,9 +36,17 @@ jobs: ${{ runner.os }}-test- ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@v1 + - name: Start redis server + run: | + echo "Starting redis server" + pwd + test/conf/redis.sh + sleep 5 + echo "Redis started" - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 + id: codecov continue-on-error: true with: files: lcov.info diff --git a/test/certs/ca.crt b/test/certs/ca.crt new file mode 100644 index 0000000..6e40c24 --- /dev/null +++ b/test/certs/ca.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIIEATCCAumgAwIBAgIUPx2cyyZHxPE/d83WJ438zA4AtDwwDQYJKoZIhvcNAQEL +BQAwgY8xCzAJBgNVBAYTAklOMQswCQYDVQQIDAJLQTESMBAGA1UEBwwJQmFuZ2Fs +b3JlMRAwDgYDVQQKDAdSZWRpc2psMRQwEgYDVQQLDAtSZWRpc2psVGVzdDEUMBIG +A1UEAwwLcmVkaXNqbHRlc3QxITAfBgkqhkiG9w0BCQEWEmNhQHJlZGlzamx0ZXN0 +LmNvbTAeFw0yMzEyMjgwOTAxMThaFw0zMzEyMjUwOTAxMThaMIGPMQswCQYDVQQG +EwJJTjELMAkGA1UECAwCS0ExEjAQBgNVBAcMCUJhbmdhbG9yZTEQMA4GA1UECgwH +UmVkaXNqbDEUMBIGA1UECwwLUmVkaXNqbFRlc3QxFDASBgNVBAMMC3JlZGlzamx0 +ZXN0MSEwHwYJKoZIhvcNAQkBFhJjYUByZWRpc2psdGVzdC5jb20wggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDc+1u5qU3MwERS98Jz1R2QnjsKD53a6wc6 +LdAli2dIEvMKymaLhCWJQpj+2dtqErJSH4S4yba4lZAV5L0+atqdECOMNPbM9jj7 +TYNCnr39Fq2Y5IAYfAqfuKxhxNUvcPbiAYRkrwd2GjkgLr9+xi53TCeESLz4oUb2 +iGd6kpe25WN5zuBr40uzq3HwSvUS2YGI5umGJGqSRAuDGj1X92VMCOGYe3AVbSwO +qvvD+vT5oO+horRzrxa9zryGsyJIo3bGnrFQz9VduTp2NGdX+RdCVULIled64x+3 +HFFQEdqBhr7lwIywiiByr6XHhSKPX5NW86aTb2PRF7/53GUUc51vAgMBAAGjUzBR +MB0GA1UdDgQWBBTWGQbX24xGF5LAAlibklwD4j31vTAfBgNVHSMEGDAWgBTWGQbX +24xGF5LAAlibklwD4j31vTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA +A4IBAQApVTxCcnKQq6XYcg7FlgtyPhkFlfyI8bGgjewUyAc2EpGjALavtdqtKk5+ +YYYT9Y1rnAE4/RGmxCVllHqmD2VBJQXf0oPOBfVTL2/S83RKKFuFxomD2+9qzdR2 +teMfLnqvZ+L36lcyHaVOiDJmtDSTnyO+Y07m5rzO80Ds/O7hQYAwtKJ72Z3arNcF +fabEQF5vB0DmSnBfRchLwwq9fkB8dZf62LSY/VvLiVXEWFbkNMQGzxOKVBSiqPSQ +7rutmP6C83DBB9z+QX3CGO1HFUlRxKpzQMEa7sQUxs0uRXfyYC7z8T52uBKoPNit +f55svM5tHApiCorGDhm109UMGzca +-----END CERTIFICATE----- diff --git a/test/certs/ca.key b/test/certs/ca.key new file mode 100644 index 0000000..2b8ca12 --- /dev/null +++ b/test/certs/ca.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDc+1u5qU3MwERS +98Jz1R2QnjsKD53a6wc6LdAli2dIEvMKymaLhCWJQpj+2dtqErJSH4S4yba4lZAV +5L0+atqdECOMNPbM9jj7TYNCnr39Fq2Y5IAYfAqfuKxhxNUvcPbiAYRkrwd2Gjkg +Lr9+xi53TCeESLz4oUb2iGd6kpe25WN5zuBr40uzq3HwSvUS2YGI5umGJGqSRAuD +Gj1X92VMCOGYe3AVbSwOqvvD+vT5oO+horRzrxa9zryGsyJIo3bGnrFQz9VduTp2 +NGdX+RdCVULIled64x+3HFFQEdqBhr7lwIywiiByr6XHhSKPX5NW86aTb2PRF7/5 +3GUUc51vAgMBAAECggEAbhHHjPFPHrrv7VGaaj7PZJ7j8m1357ikl26FXIYU0joa +FBhfvoN6fOWOtnZGS433g19OqQpUOJnKtJUETChGLvCAfFBPVCUamdXwmQjfwkX+ +/wZl5Yw3cHo4ZtR3iZNfbZBq5QmnkkSairSuTpOhRmvIzSO7+K3AXoDv5gZJil/H +vm9+jvO4QWi/lX/8MViM0FbFMyyDvY1h9owukw4iOdhvqiz6ZUofcYn8CZjkZjil +xvMY6mbeTahtxHw5OGz+2fJ7kBu/et0NSC1tOCTmewtfBW+4jDE/xXwli3OvesdV +Ec9GjecYyOI2Bm9PadRqVharelr9+Ngcl9Gwy8UmfQKBgQD1ef0DQd+Hlrp+Q30A +wXiYYVGxVxxcfcd7OTnCO6RXJUy/b+ZmzJgsBsFo8HLpH9ABo8AC8O0HsN3P56lx +nlQcGCdZIN596oYQcwvOwRBZISygu7FaS6etCdAOaheTvFm3m9AHUlkECrG5QYm1 +E35uFnrw/7+Y863JInA6xmR6gwKBgQDmdIztH6UfpLYAWaLsE7NvGhbtREZJyGxV +eM7WtPEVS7nJ23i/oKdiMhG4uTqaiF6dzrMMcDNw7BWAWOV8CdLTAJyTYVXC8oj0 +KlNGTrxNp971QjXtXfVNhbDmuiEqu/p1Z8LWNHY4QB0lErGe/8LDXkGq6EIchyvp +nLF5fSgNpQKBgGkivze27erfpq5Pb2l0c5coD1oaCh6fDGH9Z3yGGOPuKMnRPmgG +9qHcnte4LC8mnesl3CYmVA64NLhH6y2rdzC1M50b088wNA01iPgbfkn+SGPrH5Gh +14XoLwENdV2kDPTzugx4u4FcpzPGGxm/6KVz4WH413HF5EfCBTlXIU/LAoGBAMCf +RnOkcaYjTCS6x1y/kskYa3ViDcX6CeLfGTOJBcEhDGdWVHUHWSDQbNOTrxIfTcZl +UG7jEXwfOFGQ/C9THq5S4oylXMOXaTV8cyJfCTF3UPp6nwyJ7lEfn58akEJh4JRl +aAsWyoF1xWaJW4FkkWwuyoCJpUinCuM2n3jGTcYpAoGAJovcZcww0nHYzjYNgZZm +GeCzXIreAep25OlYyV18zGjAYuFBq8OP1FxrBQo66Daxa5w4N20aW522W+uskIqj +NPDtJNlonpPD/gc/R+uzhxRHLlmjrgDPrp67r/ol4i/zB9ue0SKKptx2103gZcci +atmupXm49bF30ztQUaZq1Rw= +-----END PRIVATE KEY----- diff --git a/test/certs/certgen.sh b/test/certs/certgen.sh new file mode 100755 index 0000000..3dbbe12 --- /dev/null +++ b/test/certs/certgen.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +HOSTNAME=redisjltest + +# Generate self signed root CA cert +openssl req -nodes -x509 -days 3650 -newkey rsa:2048 -keyout ca.key -out ca.crt -subj "/C=IN/ST=KA/L=Bangalore/O=Redisjl/OU=RedisjlTest/CN=${HOSTNAME}/emailAddress=ca@redisjltest.com" + +# Generate server cert to be signed +openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj "/C=IN/ST=KA/L=Bangalore/O=Redisjl/OU=RedisjlTest/CN=${HOSTNAME}/emailAddress=server@redisjltest.com" + +# Sign the server cert +openssl x509 -req -days 3650 -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt + +# Create server PEM file +cat server.key server.crt > server.pem + +# Change permissions for mounting inside container +chmod 664 *.key diff --git a/test/certs/server.crt b/test/certs/server.crt new file mode 100644 index 0000000..2307147 --- /dev/null +++ b/test/certs/server.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDqzCCApMCFCXmibnvkT2UEidAQuWBxVv1RM33MA0GCSqGSIb3DQEBCwUAMIGP +MQswCQYDVQQGEwJJTjELMAkGA1UECAwCS0ExEjAQBgNVBAcMCUJhbmdhbG9yZTEQ +MA4GA1UECgwHUmVkaXNqbDEUMBIGA1UECwwLUmVkaXNqbFRlc3QxFDASBgNVBAMM +C3JlZGlzamx0ZXN0MSEwHwYJKoZIhvcNAQkBFhJjYUByZWRpc2psdGVzdC5jb20w +HhcNMjMxMjI4MDkwMTE4WhcNMzMxMjI1MDkwMTE4WjCBkzELMAkGA1UEBhMCSU4x +CzAJBgNVBAgMAktBMRIwEAYDVQQHDAlCYW5nYWxvcmUxEDAOBgNVBAoMB1JlZGlz +amwxFDASBgNVBAsMC1JlZGlzamxUZXN0MRQwEgYDVQQDDAtyZWRpc2psdGVzdDEl +MCMGCSqGSIb3DQEJARYWc2VydmVyQHJlZGlzamx0ZXN0LmNvbTCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAKPCsgqvw+CqbThqFwDXqfRYC3NXXVuT+9Ob +wL2CwQo76TZ7gsMGFTgoH83nZN6pVp5ZNSimn8rm/U8Z3egVGEPirVaPn0Ysy6x7 +lQdPVEKISsJcezUN/EdPQgwf/S9DUBs8HuvvVT90P0kYpJVhdfduqVBPxFX018H4 +yH1Zvz0qjdDhi7YXzMJLfmK9csNVfsw7uL4Mn8ZoeU+lqOwCKt5+N55FzWu1jD8m +/SPknwcos762kiwnxluUdTLVSZs3UINVLuHipEOOljC50yaN6UWRKD8K7DFDn6lI +gKHAEXdIECseUCKPmwzHNCaLmowtGafd0EgSIqOaaJeh+YGhyxUCAwEAATANBgkq +hkiG9w0BAQsFAAOCAQEAY26Xa8q6/Ys7dN+zc666jdqfx6BPvsD/SG1x55L9eYgm +WWETj0WYj/4IC2nXSg6coElw2mFY+RBMoB6DvhzI4ARO60MV3qZtCjyooSpk+SBX +ILX1K4MpDlgrKSHEJ0hS+wOVRZk6/AkBDDKeikvcbZYoA3OdevdjSBrnEI8U1MKY +EfaL798C2sZCwW7+DqipsG3PyYQtmC26RH3Kf82ZP/ARII9Zmz0l/BQJw2IRfymI +WdpfnM/MdYDFEyKON3no69shJBWhBGFwxnRTveUYDv2QJbWgIKEmvfAxKd7no0E4 +3TqyyGoAAdiIEN4R8TY8oqCevX1l173QOOiKTpL4XQ== +-----END CERTIFICATE----- diff --git a/test/certs/server.csr b/test/certs/server.csr new file mode 100644 index 0000000..02fcc33 --- /dev/null +++ b/test/certs/server.csr @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC2TCCAcECAQAwgZMxCzAJBgNVBAYTAklOMQswCQYDVQQIDAJLQTESMBAGA1UE +BwwJQmFuZ2Fsb3JlMRAwDgYDVQQKDAdSZWRpc2psMRQwEgYDVQQLDAtSZWRpc2ps +VGVzdDEUMBIGA1UEAwwLcmVkaXNqbHRlc3QxJTAjBgkqhkiG9w0BCQEWFnNlcnZl +ckByZWRpc2psdGVzdC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQCjwrIKr8Pgqm04ahcA16n0WAtzV11bk/vTm8C9gsEKO+k2e4LDBhU4KB/N52Te +qVaeWTUopp/K5v1PGd3oFRhD4q1Wj59GLMuse5UHT1RCiErCXHs1DfxHT0IMH/0v +Q1AbPB7r71U/dD9JGKSVYXX3bqlQT8RV9NfB+Mh9Wb89Ko3Q4Yu2F8zCS35ivXLD +VX7MO7i+DJ/GaHlPpajsAirefjeeRc1rtYw/Jv0j5J8HKLO+tpIsJ8ZblHUy1Umb +N1CDVS7h4qRDjpYwudMmjelFkSg/CuwxQ5+pSIChwBF3SBArHlAij5sMxzQmi5qM +LRmn3dBIEiKjmmiXofmBocsVAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAJs2A +VQGPaQyUDtEJRrrMg0zAX0T+xPuzQBJn9xaNcbuylh1HjjOiB7msyWbugVKwcdkZ +0chAUAX0Jgslfr16GNPftBV7ddKFOfBpkg0ApjSur7WcX9fEd219Rua+avIM0e1D +a473gW5CrOja5IQnJ0ENuffXkwVPHrEtQ3E47rbPCrma37KYOJfrRHbKfCui9XTj +lHcUDknp5q/lYY2b3+XO5fqZ/BzAKFD2YoJYBWz/3i3IbDjq6daLjJQxfSZb0+sf +d8z17yV0CgfkbD8q9I8BWja7l542BnYTlTc+Me1NPupnRZC1P70iAMV8qgSypRW2 +932eP1YimsMy5tYU7A== +-----END CERTIFICATE REQUEST----- diff --git a/test/certs/server.key b/test/certs/server.key new file mode 100644 index 0000000..c6f756e --- /dev/null +++ b/test/certs/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCjwrIKr8Pgqm04 +ahcA16n0WAtzV11bk/vTm8C9gsEKO+k2e4LDBhU4KB/N52TeqVaeWTUopp/K5v1P +Gd3oFRhD4q1Wj59GLMuse5UHT1RCiErCXHs1DfxHT0IMH/0vQ1AbPB7r71U/dD9J +GKSVYXX3bqlQT8RV9NfB+Mh9Wb89Ko3Q4Yu2F8zCS35ivXLDVX7MO7i+DJ/GaHlP +pajsAirefjeeRc1rtYw/Jv0j5J8HKLO+tpIsJ8ZblHUy1UmbN1CDVS7h4qRDjpYw +udMmjelFkSg/CuwxQ5+pSIChwBF3SBArHlAij5sMxzQmi5qMLRmn3dBIEiKjmmiX +ofmBocsVAgMBAAECggEABJV+G1t5LbBHl2Us2KqwyLSwOV3QMLCPQ4igGA3xbmJn +8Ez7x47DiEtjPIXGJo9x68POf6FOZsUZgR3VvtncUIYnWP1zNWkqmFzrOoWnc6dS +BGfHUSBfl/VpgWJh2XFWbDS+HteXuMt9vAOTE9VjEnaICcX51nggk/7tScj51/Hu +CJloq8/9MnNbihJB5DgPxAt3IY2WEW93ufePGqB8QCyWTHgXvAySd+uIOpcELprR +KawUUNd/rCE0FQ79YtVZeJ5RWgPblAi0DF07WxLOfqHxUQNevpLTFL4kh/uHpeYb +htBRzfHUPtrBbWXXmHX5fUNYwicpXeBynca9soNZLwKBgQDKKJCoow9EWLRAcFuQ +98kJOqRNqdQ8Gei6drG4ECLRQcfsExSGW159bv050ebSypxayd8NeJ9R2S6guAd6 +VZjHuVhTF6uhV4jUcVx7Z37SFgS2CYWRCABoXJ70dVbSMI7wiKucLvwMFvJwVd6g +8pAs8sJ578/Md4Vj9AT1uloLrwKBgQDPYBuugPl13G5JQpSuuVghi9Z4H8W770R8 +JyEk9DYGstpfNkr+5d/zmjpZLWVN2/tAHlLPcKVW6EjqANRObKjjXB+cLfZRbr1A +GTG0FY5HVo+X0sPWY7kukX9vjHspKvy+qPVfayZn3IDJp5xwhaXc1exgZGkypujs +DCeE9soyewKBgQCVuo3KGVuJZ5m50H5BVQUVTNW8n/iNuzLgSGFAztK64lnMxCUD +jlDh3n63gHvRzDcaF0KZm6mE2bLrXuJK/XL3GpQMlw+LpGW3026ICBOqTpyWp17C +GIcUxOUGcpIng8ea598TAsmzups+EJuf4Yhfgj4ASlpCOpQVf/rcdXWUCQKBgQCD +PLxoBEFbQIQfIt67cJqAqGGzJdBabkK2G4FTRKXIOXoPVWnCxLGlFc6lTyBUVMo9 +urMHiq9oP5qdVKcHdqavNEbg7Ql8YYutPASDhjzDktlO6Nh9HiE8gmHWs13iIM1Q +z9Zxa3sjsZ1jgQp0/2+HQW7VVdZpcs3nTI3aDODLbwKBgCWZBbHSXnYUnCOQOLrA +qmj9FHoS0lhTxAHdjawd3gqLyHpZg0wNmRGliQJcfdm8KtaH8V0sUaj+Mpvm1KmS +xZGr0/V5B1AgoC+cc8eU446ahbzgq11TXvf1bqstOYFUdAb8fvvaAPudR1CtrTDu +bHGWFDcgzSgL9di4VZJ7LFCg +-----END PRIVATE KEY----- diff --git a/test/certs/server.pem b/test/certs/server.pem new file mode 100644 index 0000000..eafefbc --- /dev/null +++ b/test/certs/server.pem @@ -0,0 +1,50 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCjwrIKr8Pgqm04 +ahcA16n0WAtzV11bk/vTm8C9gsEKO+k2e4LDBhU4KB/N52TeqVaeWTUopp/K5v1P +Gd3oFRhD4q1Wj59GLMuse5UHT1RCiErCXHs1DfxHT0IMH/0vQ1AbPB7r71U/dD9J +GKSVYXX3bqlQT8RV9NfB+Mh9Wb89Ko3Q4Yu2F8zCS35ivXLDVX7MO7i+DJ/GaHlP +pajsAirefjeeRc1rtYw/Jv0j5J8HKLO+tpIsJ8ZblHUy1UmbN1CDVS7h4qRDjpYw +udMmjelFkSg/CuwxQ5+pSIChwBF3SBArHlAij5sMxzQmi5qMLRmn3dBIEiKjmmiX +ofmBocsVAgMBAAECggEABJV+G1t5LbBHl2Us2KqwyLSwOV3QMLCPQ4igGA3xbmJn +8Ez7x47DiEtjPIXGJo9x68POf6FOZsUZgR3VvtncUIYnWP1zNWkqmFzrOoWnc6dS +BGfHUSBfl/VpgWJh2XFWbDS+HteXuMt9vAOTE9VjEnaICcX51nggk/7tScj51/Hu +CJloq8/9MnNbihJB5DgPxAt3IY2WEW93ufePGqB8QCyWTHgXvAySd+uIOpcELprR +KawUUNd/rCE0FQ79YtVZeJ5RWgPblAi0DF07WxLOfqHxUQNevpLTFL4kh/uHpeYb +htBRzfHUPtrBbWXXmHX5fUNYwicpXeBynca9soNZLwKBgQDKKJCoow9EWLRAcFuQ +98kJOqRNqdQ8Gei6drG4ECLRQcfsExSGW159bv050ebSypxayd8NeJ9R2S6guAd6 +VZjHuVhTF6uhV4jUcVx7Z37SFgS2CYWRCABoXJ70dVbSMI7wiKucLvwMFvJwVd6g +8pAs8sJ578/Md4Vj9AT1uloLrwKBgQDPYBuugPl13G5JQpSuuVghi9Z4H8W770R8 +JyEk9DYGstpfNkr+5d/zmjpZLWVN2/tAHlLPcKVW6EjqANRObKjjXB+cLfZRbr1A +GTG0FY5HVo+X0sPWY7kukX9vjHspKvy+qPVfayZn3IDJp5xwhaXc1exgZGkypujs +DCeE9soyewKBgQCVuo3KGVuJZ5m50H5BVQUVTNW8n/iNuzLgSGFAztK64lnMxCUD +jlDh3n63gHvRzDcaF0KZm6mE2bLrXuJK/XL3GpQMlw+LpGW3026ICBOqTpyWp17C +GIcUxOUGcpIng8ea598TAsmzups+EJuf4Yhfgj4ASlpCOpQVf/rcdXWUCQKBgQCD +PLxoBEFbQIQfIt67cJqAqGGzJdBabkK2G4FTRKXIOXoPVWnCxLGlFc6lTyBUVMo9 +urMHiq9oP5qdVKcHdqavNEbg7Ql8YYutPASDhjzDktlO6Nh9HiE8gmHWs13iIM1Q +z9Zxa3sjsZ1jgQp0/2+HQW7VVdZpcs3nTI3aDODLbwKBgCWZBbHSXnYUnCOQOLrA +qmj9FHoS0lhTxAHdjawd3gqLyHpZg0wNmRGliQJcfdm8KtaH8V0sUaj+Mpvm1KmS +xZGr0/V5B1AgoC+cc8eU446ahbzgq11TXvf1bqstOYFUdAb8fvvaAPudR1CtrTDu +bHGWFDcgzSgL9di4VZJ7LFCg +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDqzCCApMCFCXmibnvkT2UEidAQuWBxVv1RM33MA0GCSqGSIb3DQEBCwUAMIGP +MQswCQYDVQQGEwJJTjELMAkGA1UECAwCS0ExEjAQBgNVBAcMCUJhbmdhbG9yZTEQ +MA4GA1UECgwHUmVkaXNqbDEUMBIGA1UECwwLUmVkaXNqbFRlc3QxFDASBgNVBAMM +C3JlZGlzamx0ZXN0MSEwHwYJKoZIhvcNAQkBFhJjYUByZWRpc2psdGVzdC5jb20w +HhcNMjMxMjI4MDkwMTE4WhcNMzMxMjI1MDkwMTE4WjCBkzELMAkGA1UEBhMCSU4x +CzAJBgNVBAgMAktBMRIwEAYDVQQHDAlCYW5nYWxvcmUxEDAOBgNVBAoMB1JlZGlz +amwxFDASBgNVBAsMC1JlZGlzamxUZXN0MRQwEgYDVQQDDAtyZWRpc2psdGVzdDEl +MCMGCSqGSIb3DQEJARYWc2VydmVyQHJlZGlzamx0ZXN0LmNvbTCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAKPCsgqvw+CqbThqFwDXqfRYC3NXXVuT+9Ob +wL2CwQo76TZ7gsMGFTgoH83nZN6pVp5ZNSimn8rm/U8Z3egVGEPirVaPn0Ysy6x7 +lQdPVEKISsJcezUN/EdPQgwf/S9DUBs8HuvvVT90P0kYpJVhdfduqVBPxFX018H4 +yH1Zvz0qjdDhi7YXzMJLfmK9csNVfsw7uL4Mn8ZoeU+lqOwCKt5+N55FzWu1jD8m +/SPknwcos762kiwnxluUdTLVSZs3UINVLuHipEOOljC50yaN6UWRKD8K7DFDn6lI +gKHAEXdIECseUCKPmwzHNCaLmowtGafd0EgSIqOaaJeh+YGhyxUCAwEAATANBgkq +hkiG9w0BAQsFAAOCAQEAY26Xa8q6/Ys7dN+zc666jdqfx6BPvsD/SG1x55L9eYgm +WWETj0WYj/4IC2nXSg6coElw2mFY+RBMoB6DvhzI4ARO60MV3qZtCjyooSpk+SBX +ILX1K4MpDlgrKSHEJ0hS+wOVRZk6/AkBDDKeikvcbZYoA3OdevdjSBrnEI8U1MKY +EfaL798C2sZCwW7+DqipsG3PyYQtmC26RH3Kf82ZP/ARII9Zmz0l/BQJw2IRfymI +WdpfnM/MdYDFEyKON3no69shJBWhBGFwxnRTveUYDv2QJbWgIKEmvfAxKd7no0E4 +3TqyyGoAAdiIEN4R8TY8oqCevX1l173QOOiKTpL4XQ== +-----END CERTIFICATE----- diff --git a/test/conf/redis.conf b/test/conf/redis.conf new file mode 100644 index 0000000..9c2d9af --- /dev/null +++ b/test/conf/redis.conf @@ -0,0 +1,6 @@ +port 6379 +tls-port 16379 +tls-cert-file /certs/server.pem +tls-key-file /certs/server.key +tls-ca-cert-file /certs/ca.crt +tls-auth-clients no diff --git a/test/conf/redis.sh b/test/conf/redis.sh new file mode 100755 index 0000000..10e09a0 --- /dev/null +++ b/test/conf/redis.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +SCRIPT_PATH=$(cd "$(dirname "$0")"; pwd) +TEST_PATH=$(dirname "$SCRIPT_PATH") + +docker run -d --name redis \ + --hostname redis \ + -p 6379:6379 \ + -p 16379:16379 \ + -v "${TEST_PATH}/conf/redis.conf":/usr/local/etc/redis/redis.conf \ + -v "${TEST_PATH}/certs":/certs \ + redis:7.2.3-bookworm redis-server /usr/local/etc/redis/redis.conf + From e59af60b8190ecea977c14c9ed7ae6e0a2989d58 Mon Sep 17 00:00:00 2001 From: tan Date: Thu, 28 Dec 2023 14:53:19 +0530 Subject: [PATCH 4/5] enable TLS tests --- test/runtests.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 3408b66..00c93f0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -16,5 +16,4 @@ client_tests() redis_tests(RedisConnection()) # TLS connection -# TODO: enable after updating github CI configuration -# redis_tests(RedisConnection(; port=16379, sslconfig=client_tls_config("certs/ca.crt"))) \ No newline at end of file +redis_tests(RedisConnection(; port=16379, sslconfig=client_tls_config(joinpath(@__DIR__, "certs", "ca.crt")))) \ No newline at end of file From 10dd6219d33bc913018d43af4b8ebddf31a83260 Mon Sep 17 00:00:00 2001 From: tan Date: Sat, 30 Dec 2023 22:00:05 +0530 Subject: [PATCH 5/5] address review comments --- src/transport/transport.jl | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/transport/transport.jl b/src/transport/transport.jl index b8c8038..8444209 100644 --- a/src/transport/transport.jl +++ b/src/transport/transport.jl @@ -1,16 +1,15 @@ """ - Transport module for Redis.jl abstractes the connection to the Redis server. - -Each transportimplementation must provide the following methods: -- `read_line(t::RedisTransport)`: read one line from the transport, similar to `readline` -- `read_nbytes(t::RedisTransport, m::Int)`: read `m` bytes from the transport, similar to `read` -- `write_bytes(t::RedisTransport, b::Vector{UInt8})`: write bytes to the transport, similar to `write` -- `close(t::RedisTransport)` -- `is_connected(t::RedisTransport)`: whether the transport is connected or not -- `status(t::RedisTransport)`: status of the transport, whether it is connected or not -- `set_props!(t::RedisTransport)`: set any properties required. For example, disable nagle and enable quickack to speed up the usually small exchanges -- `get_sslconfig(t::RedisTransport)`: get the SSL configuration for the transport if applicable -- `io_lock(f, t::RedisTransport)`: lock the transport for IO operations + Transport module for Redis.jl abstracts the connection to the Redis server. + +Each transport implementation must provide the following methods: +- `read_line(t::RedisTransport)`: Read one line from the transport, similar to `readline`. Return a `String`. +- `read_nbytes(t::RedisTransport, m::Int)`: Read `m` bytes from the transport, similar to `read`. Return a `Vector{UInt8}`. +- `write_bytes(t::RedisTransport, b::Vector{UInt8})`: Write bytes to the transport, similar to `write`. Return the number of bytes written. +- `close(t::RedisTransport)`: Close the transport. Return `nothing`. +- `is_connected(t::RedisTransport)`: Whether the transport is connected or not. Return a boolean. +- `set_props!(t::RedisTransport)`: Set any properties required. For example, disable nagle and enable quickack to speed up the usually small exchanges. Return `nothing`. +- `get_sslconfig(t::RedisTransport)`: Get the SSL configuration for the transport if applicable. Return a `MbedTLS.SSLConfig` or `nothing`. +- `io_lock(f, t::RedisTransport)`: Lock the transport for IO operations and execute `f`. Return the result of `f`. """ module Transport