From c342f5da7efa7f290f97628e1016724eb6306458 Mon Sep 17 00:00:00 2001 From: danda Date: Thu, 12 Jan 2017 21:20:28 -0800 Subject: [PATCH] Changes to support connecting to .onion addresses: + PeerAddress detects .onion and serializes/deserializes them using the onioncat format, which is also used by bitcoin-core, btcd, and probably others. + added some new DNS seeds from bitcoin-core + PeerGroup now listens for "addr" protocol messages and adds new peers to inactive list. This enables peer discovery beyond what is found by DNS and hard-coded seeds. Discovered peers are not presently persisted to disk. + Beginnings of a class for validating that peer addresses are routable. + Catch PeerDiscoveryException during getPeers call. avoids stack trace when all DNS lookups timeout Cherry pick https://github.com/bisq-network/bitcoinj/commit/d8a5d08fafa052394c3e97a74f521b49a4a13624 --- .../java/org/bitcoinj/core/PeerAddress.java | 72 ++++++-- .../java/org/bitcoinj/core/PeerGroup.java | 25 ++- .../java/org/bitcoinj/net/AddressChecker.java | 71 ++++++++ .../main/java/org/bitcoinj/net/OnionCat.java | 53 ++++++ .../main/java/org/bitcoinj/utils/Base32.java | 90 ++++++++++ .../java/org/bitcoinj/utils/CIDRUtils.java | 156 ++++++++++++++++++ 6 files changed, 446 insertions(+), 21 deletions(-) create mode 100644 core/src/main/java/org/bitcoinj/net/AddressChecker.java create mode 100644 core/src/main/java/org/bitcoinj/net/OnionCat.java create mode 100644 core/src/main/java/org/bitcoinj/utils/Base32.java create mode 100644 core/src/main/java/org/bitcoinj/utils/CIDRUtils.java diff --git a/core/src/main/java/org/bitcoinj/core/PeerAddress.java b/core/src/main/java/org/bitcoinj/core/PeerAddress.java index 682247602f29..704691906615 100644 --- a/core/src/main/java/org/bitcoinj/core/PeerAddress.java +++ b/core/src/main/java/org/bitcoinj/core/PeerAddress.java @@ -17,9 +17,13 @@ package org.bitcoinj.core; -import org.bitcoinj.params.MainNetParams; import com.google.common.base.Objects; import com.google.common.net.InetAddresses; +import org.bitcoinj.net.AddressChecker; +import org.bitcoinj.net.OnionCat; +import org.bitcoinj.params.MainNetParams; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.OutputStream; @@ -28,9 +32,9 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; +import static com.google.common.base.Preconditions.checkNotNull; import static org.bitcoinj.core.Utils.uint32ToByteStreamLE; import static org.bitcoinj.core.Utils.uint64ToByteStreamLE; -import static com.google.common.base.Preconditions.checkNotNull; /** *

A PeerAddress holds an IP address and port number representing the network location of @@ -39,7 +43,8 @@ *

Instances of this class are not safe for use by multiple threads.

*/ public class PeerAddress extends ChildMessage { - + private static final Logger log = LoggerFactory.getLogger(PeerAddress.class); + static final int MESSAGE_SIZE = 30; private InetAddress addr; @@ -117,7 +122,26 @@ public PeerAddress(NetworkParameters params, InetAddress addr) { * for Bitcoin. */ public PeerAddress(InetSocketAddress addr) { - this(addr.getAddress(), addr.getPort(), NetworkParameters.ProtocolVersion.CURRENT.getBitcoinProtocolVersion()); + /* socks addresses, eg Tor, use hostname only because no local lookup is performed. + * includes .onion hidden services. + */ + String host = addr.getHostString(); + if( host != null && host.endsWith(".onion") ) { + this.hostname = host; + try { + this.addr = OnionCat.onionHostToInetAddress(this.hostname); + } + catch (UnknownHostException e) { + log.warn( "Invalid format for onion address: {}", this.hostname ); + } + } + else { + this.addr = checkNotNull(addr.getAddress()); + } + this.port = addr.getPort(); + this.protocolVersion = NetworkParameters.ProtocolVersion.CURRENT.getBitcoinProtocolVersion(); + this.services = BigInteger.ZERO; + length = protocolVersion > 31402 ? MESSAGE_SIZE : MESSAGE_SIZE - 4; } /** @@ -164,14 +188,28 @@ protected void bitcoinSerializeToStream(OutputStream stream) throws IOException uint32ToByteStreamLE(secs, stream); } uint64ToByteStreamLE(services, stream); // nServices. - // Java does not provide any utility to map an IPv4 address into IPv6 space, so we have to do it by hand. - byte[] ipBytes = addr.getAddress(); - if (ipBytes.length == 4) { - byte[] v6addr = new byte[16]; - System.arraycopy(ipBytes, 0, v6addr, 12, 4); - v6addr[10] = (byte) 0xFF; - v6addr[11] = (byte) 0xFF; - ipBytes = v6addr; + + AddressChecker addrChecker = new AddressChecker(); + byte[] ipBytes; + if( addrChecker.IsOnionCatTor( addr ) ) { + ipBytes = OnionCat.onionHostToIPV6Bytes(hostname); + } + else if( addr != null ) { + // Java does not provide any utility to map an IPv4 address into IPv6 space, so we have to do it by hand. + ipBytes = addr.getAddress(); + if (ipBytes.length == 4) { + byte[] v6addr = new byte[16]; + System.arraycopy(ipBytes, 0, v6addr, 12, 4); + v6addr[10] = (byte) 0xFF; + v6addr[11] = (byte) 0xFF; + ipBytes = v6addr; + } + else { + ipBytes = new byte[16]; + } + } + else { + ipBytes = new byte[16]; // zero-filled. } stream.write(ipBytes); // And write out the port. Unlike the rest of the protocol, address and port is in big endian byte order. @@ -192,8 +230,13 @@ protected void parse() throws ProtocolException { time = -1; services = readUint64(); byte[] addrBytes = readBytes(16); + AddressChecker addrChecker = new AddressChecker(); try { addr = InetAddress.getByAddress(addrBytes); + + if( addrChecker.IsOnionCatTor( addr )) { + hostname = OnionCat.IPV6BytesToOnionHost( addr.getAddress() ); + } } catch (UnknownHostException e) { throw new RuntimeException(e); // Cannot happen. } @@ -251,7 +294,10 @@ public String toString() { if (hostname != null) { return "[" + hostname + "]:" + port; } - return "[" + addr.getHostAddress() + "]:" + port; + if(addr != null ) { + return "[" + addr.getHostAddress() + "]:" + port; + } + return "[]"; } @Override diff --git a/core/src/main/java/org/bitcoinj/core/PeerGroup.java b/core/src/main/java/org/bitcoinj/core/PeerGroup.java index 32191604e7a4..ef3e46f3b74b 100644 --- a/core/src/main/java/org/bitcoinj/core/PeerGroup.java +++ b/core/src/main/java/org/bitcoinj/core/PeerGroup.java @@ -979,22 +979,25 @@ public void addAddress(PeerAddress peerAddress) { int newMax; lock.lock(); try { - addInactive(peerAddress); - newMax = getMaxConnections() + 1; + if( addInactive(peerAddress) ) { + newMax = getMaxConnections() + 1; + setMaxConnections(newMax); + } } finally { lock.unlock(); } - setMaxConnections(newMax); } - private void addInactive(PeerAddress peerAddress) { + private boolean addInactive(PeerAddress peerAddress) { lock.lock(); try { // Deduplicate - if (backoffMap.containsKey(peerAddress)) - return; + if (backoffMap.containsKey(peerAddress)) { + return false; + } backoffMap.put(peerAddress, new ExponentialBackoff(peerBackoffParams)); inactives.offer(peerAddress); + return true; } finally { lock.unlock(); } @@ -1046,8 +1049,14 @@ protected int discoverPeers() throws PeerDiscoveryException { final List addressList = Lists.newLinkedList(); for (PeerDiscovery peerDiscovery : peerDiscoverers /* COW */) { InetSocketAddress[] addresses; - addresses = peerDiscovery.getPeers(requiredServices, peerDiscoveryTimeoutMillis, TimeUnit.MILLISECONDS); - for (InetSocketAddress address : addresses) addressList.add(new PeerAddress(params, address)); + try { + addresses = peerDiscovery.getPeers(requiredServices, peerDiscoveryTimeoutMillis, TimeUnit.MILLISECONDS); + }catch(PeerDiscoveryException e) { + log.warn(e.getMessage()); + continue; + } + + for (InetSocketAddress address : addresses) addressList.add(new PeerAddress(address)); if (addressList.size() >= maxPeersToDiscoverCount) break; } if (!addressList.isEmpty()) { diff --git a/core/src/main/java/org/bitcoinj/net/AddressChecker.java b/core/src/main/java/org/bitcoinj/net/AddressChecker.java new file mode 100644 index 000000000000..3e13a4b3d33d --- /dev/null +++ b/core/src/main/java/org/bitcoinj/net/AddressChecker.java @@ -0,0 +1,71 @@ +package org.bitcoinj.net; + +import org.bitcoinj.utils.CIDRUtils; + +import java.net.InetAddress; + +/** + * Created by danda on 1/12/17. + */ +public class AddressChecker { + + private CIDRUtils onionCatNet; + private CIDRUtils rfc4193Net; + + public AddressChecker() { + + // Note: this is borrowed/ported from btcd (written in go). + + // btcd has many more rules that are probably important and should be + // implemented in this class, but for now we only care about onion + // addresses for onioncat (ipv6) encoding/decoding. + + // onionCatNet defines the IPv6 address block used to support Tor. + // bitcoind encodes a .onion address as a 16 byte number by decoding the + // address prior to the .onion (i.e. the key hash) base32 into a ten + // byte number. It then stores the first 6 bytes of the address as + // 0xfd, 0x87, 0xd8, 0x7e, 0xeb, 0x43. + // + // This is the same range used by OnionCat, which is part part of the + // RFC4193 unique local IPv6 range. + // + // In summary the format is: + // { magic 6 bytes, 10 bytes base32 decode of key hash } + onionCatNet = new CIDRUtils("fd87:d87e:eb43::", 48); + + // rfc4193Net specifies the IPv6 unique local address block as defined + // by RFC4193 (FC00::/7). + rfc4193Net = new CIDRUtils("FC00::", 7); + } + + // IsValid returns whether or not the passed address is valid. The address is + // considered invalid under the following circumstances: + // IPv4: It is either a zero or all bits set address. + // IPv6: It is either a zero or RFC3849 documentation address. + public boolean IsValid(InetAddress addr) { + // todo: port/implement. + + // IsUnspecified returns if address is 0, so only all bits set, and + // RFC3849 need to be explicitly checked. + + // return na.IP != nil && !(na.IP.IsUnspecified() || + // na.IP.Equal(net.IPv4bcast)) + + return true; + } + + // IsOnionCatTor returns whether or not the passed address is in the IPv6 range + // used by bitcoin to support Tor (fd87:d87e:eb43::/48). Note that this range + // is the same range used by OnionCat, which is part of the RFC4193 unique local + // IPv6 range. + public boolean IsOnionCatTor(InetAddress addr) { + return onionCatNet.isInRange(addr); + } + + // IsRFC4193 returns whether or not the passed address is part of the IPv6 + // unique local range as defined by RFC4193 (FC00::/7). + public boolean IsRFC4193(InetAddress addr) { + return rfc4193Net.isInRange(addr); + } + +} diff --git a/core/src/main/java/org/bitcoinj/net/OnionCat.java b/core/src/main/java/org/bitcoinj/net/OnionCat.java new file mode 100644 index 000000000000..7081ecf9392e --- /dev/null +++ b/core/src/main/java/org/bitcoinj/net/OnionCat.java @@ -0,0 +1,53 @@ +package org.bitcoinj.net; + +import org.bitcoinj.utils.Base32; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.Arrays; + + +/** + * Created by danda on 1/12/17. + */ +public class OnionCat { + + /** Converts a .onion address to onioncat format + * + * @param hostname + * @return + */ + public static byte[] onionHostToIPV6Bytes(String hostname) { + String needle = ".onion"; + if( hostname.endsWith(needle) ) { + hostname = hostname.substring(0,hostname.length() - needle.length()); + } + byte[] prefix = new byte[] {(byte)0xfd, (byte)0x87, (byte)0xd8, (byte)0x7e, (byte)0xeb, (byte)0x43}; + byte[] onionaddr = Base32.base32Decode(hostname); + byte[] ipBytes = new byte[prefix.length + onionaddr.length]; + System.arraycopy(prefix, 0, ipBytes, 0, prefix.length); + System.arraycopy(onionaddr, 0, ipBytes, prefix.length, onionaddr.length); + + return ipBytes; + } + + public static InetAddress onionHostToInetAddress(String hostname) throws UnknownHostException { + return InetAddress.getByAddress(onionHostToIPV6Bytes(hostname)); + } + + public static InetSocketAddress onionHostToInetSocketAddress(String hostname, int port) throws UnknownHostException { + return new InetSocketAddress( onionHostToInetAddress(hostname), port); + } + + + /** Converts an IPV6 onioncat encoded address to a hostname + * + * @param bytes + * @return + */ + public static String IPV6BytesToOnionHost( byte[] bytes) { + String base32 = Base32.base32Encode( Arrays.copyOfRange(bytes, 6, 16) ); + return base32.toLowerCase() + ".onion"; + } +} diff --git a/core/src/main/java/org/bitcoinj/utils/Base32.java b/core/src/main/java/org/bitcoinj/utils/Base32.java new file mode 100644 index 000000000000..c637bc4ebea4 --- /dev/null +++ b/core/src/main/java/org/bitcoinj/utils/Base32.java @@ -0,0 +1,90 @@ +// Copied from orchid Tor lib. + +package org.bitcoinj.utils; + +import com.subgraph.orchid.TorException; + +public class Base32 { + private final static String BASE32_CHARS = "abcdefghijklmnopqrstuvwxyz234567"; + + public static String base32Encode(byte[] source) { + return base32Encode(source, 0, source.length); + } + + public static String base32Encode(byte[] source, int offset, int length) { + final int nbits = length * 8; + if(nbits % 5 != 0) + throw new TorException("Base32 input length must be a multiple of 5 bits"); + + final int outlen = nbits / 5; + final StringBuffer outbuffer = new StringBuffer(); + int bit = 0; + for(int i = 0; i < outlen; i++) { + int v = (source[bit / 8] & 0xFF) << 8; + if(bit + 5 < nbits) v += (source[bit / 8 + 1] & 0xFF); + int u = (v >> (11 - (bit % 8))) & 0x1F; + outbuffer.append(BASE32_CHARS.charAt(u)); + bit += 5; + } + return outbuffer.toString(); + } + + public static byte[] base32Decode(String source) { + int[] v = stringToIntVector(source); + + int nbits = source.length() * 5; + if(nbits % 8 != 0) + throw new TorException("Base32 decoded array must be a muliple of 8 bits"); + + int outlen = nbits / 8; + byte[] outbytes = new byte[outlen]; + + int bit = 0; + for(int i = 0; i < outlen; i++) { + int bb = bit / 5; + outbytes[i] = (byte) decodeByte(bit, v[bb], v[bb + 1], v[bb + 2]); + bit += 8; + } + return outbytes; + } + + private static int decodeByte(int bitOffset, int b0, int b1, int b2) { + switch(bitOffset % 40) { + case 0: + return ls(b0, 3) + rs(b1, 2); + case 8: + return ls(b0, 6) + ls(b1, 1) + rs (b2, 4); + case 16: + return ls(b0, 4) + rs(b1, 1); + case 24: + return ls(b0, 7) + ls(b1, 2) + rs(b2, 3); + case 32: + return ls(b0, 5) + (b1 & 0xFF); + } + throw new TorException("Illegal bit offset"); + } + + private static int ls(int n, int shift) { + return ((n << shift) & 0xFF); + } + + private static int rs(int n, int shift) { + return ((n >> shift) & 0xFF); + } + + private static int[] stringToIntVector(String s) { + final int[] ints = new int[s.length() + 1]; + for(int i = 0; i < s.length(); i++) { + int b = s.charAt(i) & 0xFF; + if(b > 0x60 && b < 0x7B) + ints[i] = b - 0x61; + else if(b > 0x31 && b < 0x38) + ints[i] = b - 0x18; + else if(b > 0x40 && b < 0x5B) + ints[i] = b - 0x41; + else + throw new TorException("Illegal character in base32 encoded string: "+ s.charAt(i)); + } + return ints; + } +} diff --git a/core/src/main/java/org/bitcoinj/utils/CIDRUtils.java b/core/src/main/java/org/bitcoinj/utils/CIDRUtils.java new file mode 100644 index 000000000000..e940a9e62cbb --- /dev/null +++ b/core/src/main/java/org/bitcoinj/utils/CIDRUtils.java @@ -0,0 +1,156 @@ +// adapted from https://github.com/edazdarevic/CIDRUtils/ +/* +* The MIT License +* +* Copyright (c) 2013 Edin Dazdarevic (edin.dazdarevic@gmail.com) +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +* THE SOFTWARE. +* +* */ + +package org.bitcoinj.utils; + +import java.math.BigInteger; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * A class that enables to get an IP range from CIDR specification. It supports + * both IPv4 and IPv6. + */ +public class CIDRUtils { + + private InetAddress inetAddress; + private InetAddress startAddress; + private InetAddress endAddress; + private final int prefixLength; + + + public CIDRUtils(String cidr) { + + try { + /* split CIDR to address and prefix part */ + if (cidr.contains("/")) { + int index = cidr.indexOf("/"); + String addressPart = cidr.substring(0, index); + String networkPart = cidr.substring(index + 1); + + inetAddress = InetAddress.getByName(addressPart); + prefixLength = Integer.parseInt(networkPart); + + calculate(); + } else { + throw new IllegalArgumentException("not an valid CIDR format!"); + } + } catch (UnknownHostException e) { + // note: no dns lookup is performed in try because these are numeric addresses. + throw new RuntimeException(e); // Cannot happen. + } + } + + public CIDRUtils(String addr, Integer prefixLength) { + + try { + inetAddress = InetAddress.getByName(addr); + this.prefixLength = prefixLength; + + calculate(); + } catch (UnknownHostException e) { + // note: no dns lookup is performed in try because these are numeric addresses. + throw new RuntimeException(e); // Cannot happen. + } + } + + private void calculate() throws UnknownHostException { + + ByteBuffer maskBuffer; + int targetSize; + if (inetAddress.getAddress().length == 4) { + maskBuffer = + ByteBuffer + .allocate(4) + .putInt(-1); + targetSize = 4; + } else { + maskBuffer = ByteBuffer.allocate(16) + .putLong(-1L) + .putLong(-1L); + targetSize = 16; + } + + BigInteger mask = (new BigInteger(1, maskBuffer.array())).not().shiftRight(prefixLength); + + BigInteger ipVal = new BigInteger(1, inetAddress.getAddress()); + + BigInteger startIp = ipVal.and(mask); + BigInteger endIp = startIp.add(mask.not()); + + byte[] startIpArr = toBytes(startIp.toByteArray(), targetSize); + byte[] endIpArr = toBytes(endIp.toByteArray(), targetSize); + + this.startAddress = InetAddress.getByAddress(startIpArr); + this.endAddress = InetAddress.getByAddress(endIpArr); + + } + + private byte[] toBytes(byte[] array, int targetSize) { + int counter = 0; + List newArr = new ArrayList(); + while (counter < targetSize && (array.length - 1 - counter >= 0)) { + newArr.add(0, array[array.length - 1 - counter]); + counter++; + } + + int size = newArr.size(); + for (int i = 0; i < (targetSize - size); i++) { + + newArr.add(0, (byte) 0); + } + + byte[] ret = new byte[newArr.size()]; + for (int i = 0; i < newArr.size(); i++) { + ret[i] = newArr.get(i); + } + return ret; + } + + public String getNetworkAddress() { + + return this.startAddress.getHostAddress(); + } + + public String getBroadcastAddress() { + return this.endAddress.getHostAddress(); + } + + public boolean isInRange(InetAddress address) { + BigInteger start = new BigInteger(1, this.startAddress.getAddress()); + BigInteger end = new BigInteger(1, this.endAddress.getAddress()); + BigInteger target = new BigInteger(1, address.getAddress()); + + int st = start.compareTo(target); + int te = target.compareTo(end); + + return (st == -1 || st == 0) && (te == -1 || te == 0); + } + + public boolean isInRange(String ipAddress) throws UnknownHostException { + return isInRange(InetAddress.getByName(ipAddress)); + } +} \ No newline at end of file