Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix some issue of network testing #1157

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Fix some issue of network testing #1157

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
761a96a
Add Ubuntu 18.04 support for AWS testing
lubaihua33 May 20, 2020
0a0a5ef
Fix some issues of TCP testing
lubaihua33 May 20, 2020
4fcf729
Add some fix when creating Azure VM
lubaihua33 Jun 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions WS2012R2/lisa/tools/middleware_bench/providers/amazon_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,12 @@ def newest_image(self, conn, os_type = None):
if os_type == 'ubuntu_1604':
filters={'name':'ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server*', 'root_device_type':'ebs', 'owner-id':'099720109477'}
log.info("ubuntu_1604")
if os_type == 'ubuntu_1804':
if self.instancetype == "m6g.4xlarge" or self.instancetype == "a1.4xlarge" or self.instancetype == "a1.metal":
filters={'name':'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-arm64-server*', 'root_device_type':'ebs', 'owner-id':'099720109477'}
else:
filters={'name':'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server*', 'root_device_type':'ebs', 'owner-id':'099720109477'}
log.info("ubuntu_1804")
elif os_type == 'amazon_linux':
filters={'name':'amzn-ami-hvm-*-x86_64-gp2', 'architecture': 'x86_64','root_device_type':'ebs'}
log.info("amazon_linux")
Expand Down Expand Up @@ -173,6 +179,9 @@ def newest_image(self, conn, os_type = None):
if os_type == 'ubuntu_1604':
self.device_map[root_device_name] = BlockDeviceType(delete_on_termination = True, size = 30, volume_type = "gp2")
log.info("device_map ubuntu_1604")
if os_type == 'ubuntu_1804':
self.device_map[root_device_name] = BlockDeviceType(delete_on_termination = True, size = 30, volume_type = "gp2")
log.info("device_map ubuntu_1804")
elif os_type == 'amazon_linux':
self.device_map[root_device_name] = BlockDeviceType(delete_on_termination = True, size = 30, volume_type = "gp2")
log.info("device_map amazon_linux")
Expand Down
2 changes: 1 addition & 1 deletion WS2012R2/lisa/tools/middleware_bench/providers/azure_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ def connect(self):
self.location))
self.resource_client.resource_groups.create_or_update(self.group_name,
{'location': self.location})
if self.instancetype == 'Standard_NC6':
if self.instancetype == 'Standard_NC6' or self.instancetype == 'Standard_D16_v3' or self.instancetype == 'Standard_D64_v3' or self.instancetype == 'Standard_E16_v3':
sku = 'standard_lrs'
else:
sku = 'premium_lrs'
Expand Down
2 changes: 1 addition & 1 deletion WS2012R2/lisa/tools/middleware_bench/report/results_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -988,7 +988,7 @@ def collect_data(self, f_match, log_file, log_dict):
with open(log_file, 'r') as fl:
for x in fl:
if not log_dict.get('Throughput_Gbps', None):
throughput = re.match('.+throughput.+:([0-9.]+)', x)
throughput = re.match('.+INFO:.+throughput.+:([0-9.]+)', x)
if throughput:
log_dict['Throughput_Gbps'] = throughput.group(1).strip()
if not log_dict.get('PacketSize_KBytes', None):
Expand Down
28 changes: 14 additions & 14 deletions WS2012R2/lisa/tools/middleware_bench/tests/run_network.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,17 +47,17 @@ then
sudo apt install -y aptitude
printf '.\n.\n.\n.\nY\nY\n' |sudo aptitude install build-essential >> ${LOG_FILE}
sudo apt -y install build-essential >> ${LOG_FILE}
sudo apt -y install sysstat zip bc >> ${LOG_FILE}
sudo apt -y install sysstat zip bc cmake>> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo apt update"
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo apt install -y aptitude"
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "printf '.\n.\n.\n.\nY\nY\n' |sudo aptitude install build-essential"
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo apt -y install sysstat zip bc build-essential" >> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo apt -y install sysstat zip bc build-essential cmake" >> ${LOG_FILE}
elif [[ ${distro} == *"Amazon"* ]]
then
sudo yum clean dbcache >> ${LOG_FILE}
sudo yum -y install sysstat zip bc git gcc automake autoconf rpm >> ${LOG_FILE}
sudo yum -y install sysstat zip bc git gcc automake autoconf rpm cmake>> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo yum clean dbcache" >> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo yum -y install sysstat zip bc git gcc automake autoconf rpm" >> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo yum -y install sysstat zip bc git gcc automake autoconf rpm cmake" >> ${LOG_FILE}
else
LogMsg "Unsupported distribution: ${distro}."
fi
Expand All @@ -72,16 +72,16 @@ then
cd /tmp/ntttcp-for-linux/src; sudo make && sudo make install
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "cd /tmp; git clone https://github.com/Microsoft/ntttcp-for-linux" >> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "cd /tmp/ntttcp-for-linux/src; sudo make && sudo make install" >> ${LOG_FILE}
cd /tmp; git clone https://github.com/Microsoft/lagscope
cd /tmp/lagscope/src; sudo make && sudo make install
cd /tmp; git clone https://github.com/Microsoft/lagscope >> ${LOG_FILE}
cd /tmp/lagscope; sudo ./do-cmake.sh build && sudo ./do-cmake.sh install >> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "cd /tmp; git clone https://github.com/Microsoft/lagscope" >> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "cd /tmp/lagscope/src; sudo make && sudo make install" >> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "cd /tmp/lagscope; sudo ./do-cmake.sh build && sudo ./do-cmake.sh install" >> ${LOG_FILE}
elif [[ ${TEST_TYPE} == "latency" ]]
then
cd /tmp; git clone https://github.com/Microsoft/lagscope
cd /tmp/lagscope/src; sudo make && sudo make install
cd /tmp/lagscope; sudo ./do-cmake.sh build && sudo ./do-cmake.sh install >> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "cd /tmp; git clone https://github.com/Microsoft/lagscope" >> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "cd /tmp/lagscope/src; sudo make && sudo make install" >> ${LOG_FILE}
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "cd /tmp/lagscope; sudo ./do-cmake.sh build && sudo ./do-cmake.sh install" >> ${LOG_FILE}
elif [[ ${TEST_TYPE} == "UDP" ]]
then
TEST_THREADS=(1 2 4 8 16 32 64 128 256 512 1024)
Expand Down Expand Up @@ -166,7 +166,7 @@ function run_lagscope()
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo pkill -f lagscope"
ssh -f -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo lagscope -r${SERVER}"
sleep 5
sudo lagscope -s${SERVER} -n1000000 -i0 -V > "/tmp/network${TEST_TYPE}/lagscope.log"
sudo lagscope -s${SERVER} -n1000000 -i0 -H > "/tmp/network${TEST_TYPE}/lagscope.log"
sleep 5
sudo pkill -f lagscope
}
Expand All @@ -187,15 +187,15 @@ function run_ntttcp ()
fi
sudo pkill -f ntttcp
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo pkill -f ntttcp"
ssh -f -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo ntttcp -r${SERVER} -P $num_threads_P -e > /tmp/network${TEST_TYPE}/${current_test_threads}_ntttcp-receiver.log"
ssh -f -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo ntttcp -r${SERVER} -P $num_threads_P -e -W 1 -C 1 > /tmp/network${TEST_TYPE}/${current_test_threads}_ntttcp-receiver.log"
sudo pkill -f lagscope
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo pkill -f lagscope"
ssh -f -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo lagscope -r${SERVER}"
sleep 5
previous_tx_bytes=$(get_tx_bytes)
previous_tx_pkts=$(get_tx_pkts)
sudo lagscope -s${SERVER} -t 60 -V 4 > "/tmp/network${TEST_TYPE}/${current_test_threads}_lagscope.log"
sudo ntttcp -s${SERVER} -P ${num_threads_P} -n ${num_threads_n} -t 60 > "/tmp/network${TEST_TYPE}/${current_test_threads}_ntttcp-sender.log"
sudo lagscope -s${SERVER} -t60 > "/tmp/network${TEST_TYPE}/${current_test_threads}_lagscope.log" &
sudo ntttcp -s${SERVER} -P ${num_threads_P} -n ${num_threads_n} -t 60 -W 1 -C 1 > "/tmp/network${TEST_TYPE}/${current_test_threads}_ntttcp-sender.log"
current_tx_bytes=$(get_tx_bytes)
current_tx_pkts=$(get_tx_pkts)
bytes_new=`(expr ${current_tx_bytes} - ${previous_tx_bytes})`
Expand Down Expand Up @@ -304,7 +304,7 @@ function run_custom()
if [[ ${TEST_TYPE} == "TCP" ]]
then
ulimit -n 204800
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "sudo ulimit -n 204800"
ssh -o StrictHostKeyChecking=no ${USER}@${SERVER} "ulimit -n 204800"
for thread in "${TEST_THREADS[@]}"
do
run_ntttcp ${thread}
Expand Down
5 changes: 3 additions & 2 deletions WS2012R2/lisa/tools/middleware_bench/utils/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,8 @@ def get_instance_details(self):
self.connector.localpath,
self.connector.key_name + '.pem'))
ip = ssh_client[i].run(
'ifconfig eth0 | grep "inet\ addr" | cut -d: -f2 | cut -d" " -f1')
'ifconfig eth0 | grep "inet" | cut -d: -f2 | awk -F " " \'{print $2}\' | head -n 1')
log.info('vm ip {}'.format(ip))
vm_ips[i] = ip[1].strip()
elif self.provider == constants.GCE:
ssh_client[i] = self.connector.wait_for_ping(self.vms[i])
Expand Down Expand Up @@ -268,7 +269,7 @@ def perf_tuning(self):
self.connector.localpath,
self.connector.key_name + '.pem'))
ip = self.ssh_client[i].run(
'ifconfig eth0 | grep "inet\ addr" | cut -d: -f2 | cut -d" " -f1')
'ifconfig eth0 | grep "inet" | cut -d: -f2 | awk -F " " \'{print $2}\' | head -n 1')
self.vm_ips[i] = ip[1].strip()

def run_test(self, ssh_vm_conf=0, testname=None, test_cmd=None, results_path=None, raid=False,
Expand Down