Page History

#!/usr/bin/python3

import sys
import subprocess
import datetime
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
import json
import pingparsing

def return_command_output(command):
    proc = subprocess.Popen(command, stdout = subprocess.PIPE, shell = True)
    (out, err) = proc.communicate()
    output = out.rstrip('\n'.encode('utf8'))
    return output

def get_mac(iface):
    command = "cat /sys/class/net/" + str(iface) + "/address"
    mac = 

return_command_output(command)

.decode('utf8')
    

mac = 

mac.

replace(

":", "-")
    

return mac

def find_wlan_iface_name():
    

command = 

"printf '%s\n' /sys/class/net/*/wireless | awk -F'/' '{print $5 }'"
    wlan_iface_name = return_command_output(command)
    return 

wlan_iface_name.decode('utf8')

def 

parse_

iwconfig(iface):
    

bit_rate = 

return_command_output("sudo iwconfig " + 

iface

 + 

"

 | grep Bit | 

awk 

'{print $2}' | sed 's/Rate=//'").decode('utf8')
    

tx_power = 

return

_command_output("sudo iwconfig " + iface + " | grep Bit | awk '{print 

$4}'

 | 

sed 's/Tx-Power=//'").decode('utf8')

link_

quality = return_command_output("sudo iwconfig " + iface + " | grep 

Link | awk '{print $2}' | sed 's/

Quality=//'").decode('utf8')
    

link_quality = link_quality.split("/")[0]
    signal_level = return_command_output("sudo iwconfig " + iface + " | grep 

Link | awk '{print $4}' | sed 's/

level=//'").decode('utf8')
    

accesspoint = return_command_output("sudo iwconfig " + iface + " | grep 

Mode | awk '{print 

$6}' | sed 's/

Point: //'").decode('utf8')
    

accesspoint = 

accesspoint.

replace("

:", "-")
    

essid = return_command_output("sudo iwconfig " + iface + " | grep 

ESSID | awk '{print $4}' | sed 's/

ESSID://'").decode('utf8')
    

essid = 

essid.replace("

\"", "")
    return bit_rate, tx_power, link_quality, signal_level, accesspoint, essid

def parse_iwlist(iface, accesspoint):
    

information = 

{}
    

command = 

"sudo 

iwlist " + iface + " scan | grep 

-E \"Cell|Frequency|Quality|ESSID\""
    aps = return_command_output(command).decode(

"utf8

")
    

aps = 

aps.

split("\

n")

    

cell_indices = list()
    for index in range(0, len(aps)):
        line_no_whitespace = ' '.join(aps[index].split())
        parts = line_no_whitespace.split()
        if parts[0] == "Cell":
            cell_indices.append(index)

    

for index in 

cell_indices:
        

line0 = ' '.join(aps[index].split())
        

ap_mac = 

line0.split()[-1]
        

ap_mac = ap_mac.replace(":", "-")
        information[ap_mac] 

= {}
        

line1 = ' '.join(aps[index + 1].split())
        

frequency = 

line1.split()[

0].split(":

")[1]
        information[ap_mac]["frequency"] = 

str(frequency)
        

line2 = ' '.join(aps[index + 

2].split())
        

parts = 

line2.split()

        information[ap_mac]["

drillTest"] = 

float(parts[2].split(

"

=

"

)[1])
        line3 = ' '.join(aps[index + 3].split())
        parts = line3.split(":")
        information[ap_mac][str(parts[1].replace('"', ''))] = information[ap_mac]["drillTest"]

    return information

def convert_info_to_json(accesspoint, essid, mac, bit_rate, tx_power, link_quality, signal_level, probe_no, information, location_name, test_device_location_description, nat_network, system_dictionary, number_of_users, pingparser_result):
    overall_dictionary = {}
    # values from ping received through pingparser github tool
    overall_dictionary["wts"] = str(pingparser_result["destination"])
    packet_transmit = int(float(pingparser_result["packet_transmit"]))
    overall_dictionary["pingPacketTransmit"] = str(packet_transmit)
    packet_receive = int(float(pingparser_result["packet_receive"]))
    overall_dictionary["pingPacketReceive"] = str(packet_receive)
    packet_loss_rate = int(float(pingparser_result["packet_loss_rate"]))
    overall_dictionary["pingPacketLossRate"] = str(packet_loss_rate)
    packet_loss_count = int(float(pingparser_result["packet_loss_count"]))
    overall_dictionary["pingPacketLossCount"] = str(packet_loss_count)
    try:
        rtt_min = int(float(pingparser_result["rtt_min"]))
        rtt_avg = int(float(pingparser_result["rtt_avg"]))
        rtt_max = int(float(pingparser_result["rtt_max"]))
        rtt_mdev = int(float(pingparser_result["rtt_mdev"]))
        packet_duplicate_rate = int(float(pingparser_result["packet_duplicate_rate"]))
        packet_duplicate_count = int(float(pingparser_result["packet_duplicate_count"]))
    except:
        # -1 indicates failure to reach the wts and calculate the above values
        rtt_min = -1
        rtt_avg = -1
        rtt_max = -1
        rtt_mdev = -1
        packet_duplicate_rate = -1
        packet_duplicate_count = -1
    overall_dictionary["pingRttMin"] = str(rtt_min)
    overall_dictionary["pingRttAvg"] = str(rtt_avg)
    overall_dictionary["pingRttMax"] = str(rtt_max)
    overall_dictionary["pingRttMdev"] = str(rtt_mdev)
    overall_dictionary["pingPacketDuplicateRate"] = str(packet_duplicate_rate)
    overall_dictionary["pingPacketDuplicateCount"] = str(packet_duplicate_count)
    # values from iw* commands
    overall_dictionary["macAddress"] = "\"" + str(mac) + "\""
    overall_dictionary["accesspoint"] = "\"" + str(accesspoint) + "\""
    overall_dictionary["essid"] = "\"" + str(essid) + "\""
    bit_rate = int(float(bit_rate))
    overall_dictionary["bitRate"] = str(bit_rate)
    tx_power = int(float(tx_power))
    overall_dictionary["txPower"] = str(tx_power)
    link_quality = int(float(link_quality))
    overall_dictionary["linkQuality"] = str(link_quality)
    signal_level = int(float(signal_level))
    overall_dictionary["signalLevel"] = str(signal_level)
    overall_dictionary["probeNo"] = str(probe_no)
    information = json.dumps(information)
    overall_dictionary["monitor"] = information
    # values defined by administrator
    overall_dictionary["locationName"] = "\"" + str(location_name) + "\""
    overall_dictionary["testDeviceLocationDescription"] = "\"" + str(test_device_location_description) + "\""
    overall_dictionary["nat"] = "\"" + str(nat_network) + "\""
    # values received through arp-scan command
    overall_dictionary["numberOfUsers"] = "\"" + str(number_of_users) + "\""
    system_dictionary = json.dumps(system_dictionary)
    # values received from system commands (memory, cpu, disk)
    overall_dictionary["system"] = system_dictionary
    json_data = json.dumps(overall_dictionary)
    return json_data

def processing_info():
    command = '''echo "$(iostat | head -1 | awk '{print $1}')"'''
    operating_system = return_command_output(command).decode('utf8')
    command = '''echo "$(iostat | head -1 | awk '{print $2}')"'''
    driver_version = return_command_output(command).decode('utf8')
    command = '''echo "$(iostat | head -1 | awk '{print $6}' | cut -c 2-)"'''
    total_cores = return_command_output(command).decode('utf8')
    command = '''echo "$(vmstat 1 2|tail -1|awk '{print $15}')"'''
    cpu_utilization = 100 - int(return_command_output(command).decode('utf8'))
    command = '''echo "$(vmstat --stats | grep 'total memory' | tail -1 | awk '{print $1}')"'''
    total_memory = return_command_output(command).decode('utf8')
    command = '''echo "$(vmstat --stats | grep 'used memory' | tail -1 | awk '{print $1}')"'''
    used_memory = return_command_output(command).decode('utf8')
    command = '''echo "$(df -h / | tail -1 | awk '{print $2}')"'''
    total_disk_size = return_command_output(command).decode('utf8')
    command = '''echo "$(df -h / | tail -1 | awk '{print $3}')"'''
    used_disk_size = return_command_output(command).decode('utf8')

    system_dictionary = {}
    system_dictionary["operatingSystem"] = str(operating_system)
    system_dictionary["driverVersion"] = str(driver_version)
    system_dictionary["totalCores"] = str(total_cores) 
    system_dictionary["cpuUtilization"] = str(cpu_utilization)
    system_dictionary["totalMemory"] = str(total_memory) 
    system_dictionary["usedMemory"] = str(used_memory)
    system_dictionary["totalDiskSize"] = str(total_disk_size) 
    system_dictionary["usedDiskSize"] = str(used_disk_size)

    return system_dictionary

def stream_data(data):
    headers = {'content-type':"application/json"}
    try:
        session = requests.Session()
        session.verify = False
        session.post(url='https://INSERT_WAS_FQDN:443/wifimon/probes/', data=data, headers=headers, timeout=30)
    except:
        pass

def parse_arpscan(result):
    lines = result.split("\n")
    lines.pop(0)
    lines.pop(0)
    space_line = lines.index('')
    return space_line

def arpscanner():
    command = "sudo arp-scan --localnet"
    arpscan_result = return_command_output(command).decode('utf8')
    number_of_users = parse_arpscan(arpscan_result)
    return number_of_users

def pingparser(wts):
    # See: https://github.com/thombashi/pingparsing
    ping_parser = pingparsing.PingParsing()
    transmitter = pingparsing.PingTransmitter()
    transmitter.destination = str(wts)
    transmitter.count = 3
    result = transmitter.ping()
    result_json = json.dumps(ping_parser.parse(result).as_dict(), indent=4)
    result_dict = json.loads(result_json)
    return result_dict

def set_location_information():
    location_name = "INSERT_LOCATION_NAME"
    test_device_location_description = "INSERT_TEST_DEVICE_LOCATION_DESCRIPTION"
    nat_network = "INSERT_True_OR_False"
    return location_name, test_device_location_description, nat_network

def general_info():
    system_dictionary = processing_info()
    location_name, test_device_location_description, nat_network = set_location_information()
    iface_name = find_wlan_iface_name()
    mac = get_mac(iface_name)
    bit_rate, tx_power, link_quality, signal_level, accesspoint, essid = parse_iwconfig(iface_name)
    information = parse_iwlist(iface_name, accesspoint)
    probe_no = "INSERT_PROBE_NUMBER"
    wts = "INSERT_WTS_FQDN"
    number_of_users = arpscanner()
    pingparser_result = pingparser(wts)
    json_data = convert_info_to_json(accesspoint, essid, mac, bit_rate, tx_power, link_quality, signal_level, probe_no, information, location_name, test_device_location_description, nat_network, system_dictionary, number_of_users, pingparser_result)
    stream_data(json_data)

if

__name__ == "__main__":
    general_info()

'''
Sample twping output (MIND THE NAMING OF THE LINES)

line 0: --- twping statistics from [192.168.1.1]:9706 to [192.168.1.2]:19642 ---
line 1: SID:    c0a80102e5e36a42b8a73f74cec8780e
line 2: first:  2022-03-21T23:18:58.819
line 3: last:   2022-03-21T23:19:10.456
line 4: 100 sent, 0 lost (0.000%), 0 send duplicates, 0 reflect duplicates
line 5: round-trip time min/median/max = 0.109/0.3/1.07 ms, (err=3.8 ms)
line 6: send time min/median/max = 936/936/936 ms, (err=1.9 ms)
line 7: reflect time min/median/max = -936/-936/-935 ms, (err=1.9 ms)
line 8: reflector processing time min/max = 0.00191/0.021 ms
line 9: two-way jitter = 0.1 ms (P95-P50)
line 10: send jitter = 0.1 ms (P95-P50)
line 11: reflect jitter = 0 ms (P95-P50)
line 12: send hops = 0 (consistently)
line 13:reflect hops = 0 (consistently)
'''

import subprocess
import json
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

def return_command_output(command):
    '''
        Execute a command and return its output
    '''
    proc = subprocess.Popen(command, stdout = subprocess.PIPE, shell = True)
    (out, err) = proc.communicate()
    output = out.rstrip('\n'.encode('utf8'))
    return output

def perform_twping(twamp_server_ip):
    '''
        Perform the twping command and retrieve its output in milliseconds
    '''
    command = "twping " + str(twamp_server_ip) + " -n m"
    twping_results = return_command_output(command).decode('utf8')
    return twping_results

def locate_twping_data(twping_output):
    '''
        Find the line at which the important part of the twping output starts
    '''
    twping_output_parts = twping_output.split('\n')
    line_to_start = 0
    for line in twping_output_parts:
        initial_three_chars = line[0:3]
        if initial_three_chars == "---":
            break
        line_to_start += 1
    return line_to_start

# Parse lines one by one. Look at the top for the numbering of the lines
def parse_line1(line1):
    parts = line1.split("\t")
    sid = parts[1]
    return sid

def parse_line4(line4):
    parts = line4.split(" ")
    sent, lost, send_dups, reflect_dups = parts[0], parts[2], parts[5], parts[8]
    return sent, lost, send_dups, reflect_dups

def parse_times(line):
    parts = line.split(" ")
    min_median_max = parts[4].split("/")
    minimum, median, maximum = min_median_max[0], min_median_max[1], min_median_max[2]
    err = parts[6].split("=")[1]
    return minimum, median, maximum, err

def parse_line8(line):
    parts = line.split(" ")
    time_unit = parts[-1]
    minimum = parts[-2].split("/")[0]
    maximum = parts[-2].split("/")[1]
    return minimum, maximum

def parse_jitter(line):
    parts = line.split(" ")
    value = parts[3]
    characterization = parts[5][1:-1]
    return value, characterization

def parse_hops(line):
    parts = line.split(" ")
    value = parts[3]
    characterization = parts[4][1:-1]
    return value, characterization

def parse_ntpstat_line_0(line):
    line_parts = line.split(" ")
    ntp_server = line_parts[4]
    ntp_server = ntp_server[1:-1]
    stratum = line_parts = line_parts[7]
    return ntp_server, stratum

def parse_ntpstat_line_1(line):
    line_parts = line.split(" ")
    while line_parts[0] == "":
        line_parts = line_parts[1:]
    value = line_parts[4]
    unit = line_parts[5]
    time_correct = str(value) + " " + str(unit)
    return time_correct

def parse_ntpstat():
    command = "ntpstat"
    ntpstat_output = return_command_output(command).decode('utf8')
    ntpstat_output_lines = ntpstat_output.split('\n')
    line_0 = ntpstat_output_lines[0]
    ntp_server, stratum = parse_ntpstat_line_0(line_0)
    line_1 = ntpstat_output_lines[1]
    time_correct = parse_ntpstat_line_1(line_1)
    return (ntp_server, stratum, time_correct)

def parse_ntpq_starred_line(line):
    line_parts = line.split(" ")
    try:
        while True:
            line_parts.remove('')
    except ValueError:
        pass
    return line_parts

def parse_ntpq():
    command = "ntpq -pn"
    ntpq_output = return_command_output(command).decode('utf8')
    ntpq_output_lines = ntpq_output.split('\n')
    for line in ntpq_output_lines:
        if line[0] == "*":
            ntpq_result = parse_ntpq_starred_line(line[1:])
    return ntpq_result

def form_json(probe_number, twamp_server, sid, sent, lost, send_dups, reflect_dups, 
        min_rtt, median_rtt, max_rtt, err_rtt, min_send, median_send, max_send, 
        err_send, min_reflect, median_reflect, max_reflect, err_reflect, 
        min_reflector_processing_time, max_reflector_processing_time,
        two_way_jitter_value, two_way_jitter_char, send_jitter_value, send_jitter_char,
        reflect_jitter_value, reflect_jitter_char, send_hops_value, send_hops_char,
        reflect_hops_value, reflect_hops_char, ntp_server_ntpstat, stratum, time_correct,
        ntp_server_ntpq, delay_ntpq, offset_ntpq, jitter_ntpq):
    '''
        Create a json object with the parsed values. Values are first stored in a dictionary.
    '''
    overall_dictionary = {}
    # TWAMP-related data
    overall_dictionary["probeNumber"] = probe_number
    overall_dictionary["twampServer"] = twamp_server
    overall_dictionary["sid"] = sid
    overall_dictionary["sent"] = sent
    overall_dictionary["lost"] = lost
    overall_dictionary["sendDups"] = send_dups
    overall_dictionary["reflectDups"] = reflect_dups
    overall_dictionary["minRtt"] = min_rtt
    overall_dictionary["medianRtt"] = median_rtt
    overall_dictionary["maxRtt"] = max_rtt
    overall_dictionary["errRtt"] = err_rtt
    overall_dictionary["minSend"] = min_send
    overall_dictionary["medianSend"] = median_send
    overall_dictionary["maxSend"] = max_send
    overall_dictionary["errSend"] = err_send
    overall_dictionary["minReflect"] = min_reflect
    overall_dictionary["medianReflect"] = median_reflect
    overall_dictionary["maxReflect"] = max_reflect
    overall_dictionary["errReflect"] = err_reflect
    overall_dictionary["minReflectorProcessingTime"] = min_reflector_processing_time
    overall_dictionary["maxReflectorProcessingTime"] = max_reflector_processing_time
    overall_dictionary["twoWayJitterValue"] = two_way_jitter_value
    overall_dictionary["twoWayJitterChar"] = two_way_jitter_char
    overall_dictionary["sendJitterValue"] = send_jitter_value
    overall_dictionary["sendJitterChar"] = send_jitter_char
    overall_dictionary["reflectJitterValue"] = reflect_jitter_value
    overall_dictionary["reflectJitterChar"] = reflect_jitter_char
    overall_dictionary["sendHopsValue"] = send_hops_value
    overall_dictionary["sendHopsChar"] = send_hops_char
    overall_dictionary["reflectHopsValue"] = reflect_hops_value
    overall_dictionary["reflectHopsChar"] = reflect_hops_char
    # NTP-related data
    overall_dictionary["ntpServerNtpstat"] = "\"" + str(ntp_server_ntpstat) + "\""
    overall_dictionary["stratum"] = stratum
    overall_dictionary["timeCorrect"] = time_correct
    overall_dictionary["ntpServerNtpq"] = "\"" + str(ntp_server_ntpq) + "\""
    overall_dictionary["delayNtpq"] = delay_ntpq
    overall_dictionary["offsetNtpq"] = offset_ntpq
    overall_dictionary["jitterNtpq"] = jitter_ntpq
    json_data = json.dumps(overall_dictionary)
    return json_data

def parse_twping_and_ntp(twping_output, line_to_start, probe_number):
    '''
        Parse twping output line by line
    '''
    twping_output_parts = twping_output.split('\n')
    sid = parse_line1(twping_output_parts[line_to_start + 1])
    sent, lost, send_dups, reflect_dups = parse_line4(twping_output_parts[line_to_start + 4])
    min_rtt, median_rtt, max_rtt, err_rtt = parse_times(twping_output_parts[line_to_start + 5])
    min_send, median_send, max_send, err_send = parse_times(twping_output_parts[line_to_start + 6])
    min_reflect, median_reflect, max_reflect, err_reflect = parse_times(twping_output_parts[line_to_start + 7])
    min_reflector_processing_time, max_reflector_processing_time = parse_line8(twping_output_parts[line_to_start +8])
    two_way_jitter_value, two_way_jitter_char = parse_jitter(twping_output_parts[line_to_start + 9])
    send_jitter_value, send_jitter_char = parse_jitter(twping_output_parts[line_to_start + 10])
    reflect_jitter_value, reflect_jitter_char = parse_jitter(twping_output_parts[line_to_start + 11])
    send_hops_value, send_hops_char = parse_hops(twping_output_parts[line_to_start + 12])
    reflect_hops_value, reflect_hops_char = parse_hops(twping_output_parts[line_to_start + 13])
    # parse ntpq and ntpstat commands
    ntp_server_ntpstat, stratum, time_correct = parse_ntpstat()
    ntpq_result = parse_ntpq()
    ntp_server_ntpq = ntpq_result[0]
    delay_ntpq = ntpq_result[7]
    offset_ntpq = ntpq_result[8]
    jitter_ntpq = ntpq_result[9]
    # form json data
    json_data = form_json(probe_number, twamp_server, sid, sent, lost, send_dups, reflect_dups, 
            min_rtt, median_rtt, max_rtt, err_rtt, min_send, median_send, max_send, err_send, 
            min_reflect, median_reflect, max_reflect, err_reflect, min_reflector_processing_time,
            max_reflector_processing_time, two_way_jitter_value, two_way_jitter_char, 
            send_jitter_value, send_jitter_char, reflect_jitter_value, reflect_jitter_char, 
            send_hops_value, send_hops_char, reflect_hops_value, reflect_hops_char,
            ntp_server_ntpstat, stratum, time_correct, ntp_server_ntpq, delay_ntpq,
            offset_ntpq, jitter_ntpq)
    return json_data

def stream_data(json_data):
    '''
        Stream JSON data to the WiFiMon Analysis Server
        Set the FQDN of the WiFiMon Analysis Server
    '''
    headers = {'content-type' : "application/json"}
    try:
        session = requests.Session()
        session.verify = False
        session.post(url = 'https://INSERT_WAS_FQDN_OR_IP:443/wifimon/twamp/', data = json_data, headers = headers, timeout = 30)
    except:
        pass
    return None

if __name__ == "__main__":
    # Define the number of the WiFiMon Hardware Probe
    PROBE_NO = "INSERT_PROBE_NUMBER"
    # Define the IP address of the TWAMP Server
    twamp_server = "INSERT_TWAMP_SERVER_FQDN_OR_IP"
    twping_results = perform_twping(twamp_server)
    line_to_start = locate_twping_data(twping_results)
    json_data = parse_twping_and_ntp(twping_results, line_to_start, PROBE_NO)
    stream_data(json_data)