allow save and load of binary protocols (#488)

* allow save and load of binary protocols

* update changelog

* use vs2017 for appveyor

* update requirements

* update travis

.travis.yml

@@ -46,7 +46,7 @@ install:
     then
       sudo apt-get update && sudo apt-get -y install python3-zmq python3-pyqt5 python3-numpy python3-psutil
     else
-      pip3 install pyqt5!=5.10 && pip3 install -r data/requirements.txt
+      pip3 install -r data/requirements.txt
     fi
   - |
     if [[ $TRAVIS_PYTHON_VERSION == "3.6"* ]]

CHANGELOG.md

@@ -1,4 +1,10 @@
 # Changelog
+## v2.2.3 (upcoming)
+New features:
+- allow save and load of binary protocols (``` .bin ``` files) [#488](https://github.com/jopohl/urh/pull/488)
+
+---
+
 ## v2.2.2 (01/07/2018)
 This release removes the ``` config.pxi ``` requirement which caused problems on Arch Linux and Gentoo during installation. More details in PR [#484](https://github.com/jopohl/urh/pull/484).
 

appveyor.yml

@@ -46,7 +46,7 @@ install:
   - 7z x -y "C:\windlls.zip" -o%APPVEYOR_BUILD_FOLDER%\src\urh\dev\native\lib\shared
 
   - pip install -r data\requirements.txt
-  - pip install pytest
+  - pip install pytest pytest-faulthandler
 
 build_script:
   # Build the compiled extension

data/requirements.txt

@@ -1,5 +1,6 @@
 numpy
 psutil
-pyqt5!=5.11
+pyqt5; sys_platform != 'win32'
+pyqt5!=5.11.1,!=5.11.2; sys_platform == 'win32'
 pyzmq
 cython

src/urh/controller/CompareFrameController.py

@@ -424,19 +424,18 @@ def add_protocol(self, protocol: ProtocolAnalyzer, group_id: int = 0) -> Protoco
         return protocol
 
     def add_protocol_from_file(self, filename: str) -> ProtocolAnalyzer:
-        """
-
-        :rtype: list of ProtocolAnalyzer
-        """
         pa = ProtocolAnalyzer(signal=None, filename=filename)
         pa.message_types = []
 
-        pa.from_xml_file(filename=filename, read_bits=True)
-        for messsage_type in pa.message_types:
-            if messsage_type not in self.proto_analyzer.message_types:
-                if messsage_type.name in (mt.name for mt in self.proto_analyzer.message_types):
-                    messsage_type.name += " (" + os.path.split(filename)[1].rstrip(".xml").rstrip(".proto") + ")"
-                self.proto_analyzer.message_types.append(messsage_type)
+        if filename.endswith(".bin"):
+            pa.from_binary(filename)
+        else:
+            pa.from_xml_file(filename=filename, read_bits=True)
+            for messsage_type in pa.message_types:
+                if messsage_type not in self.proto_analyzer.message_types:
+                    if messsage_type.name in (mt.name for mt in self.proto_analyzer.message_types):
+                        messsage_type.name += " (" + os.path.split(filename)[1].rstrip(".xml").rstrip(".proto") + ")"
+                    self.proto_analyzer.message_types.append(messsage_type)
 
         self.fill_message_type_combobox()
         self.add_protocol(protocol=pa)
@@ -837,8 +836,11 @@ def save_protocol(self):
         if not filename:
             return
 
-        self.proto_analyzer.to_xml_file(filename=filename, decoders=self.decodings,
-                                        participants=self.project_manager.participants, write_bits=True)
+        if filename.endswith(".bin"):
+            self.proto_analyzer.to_binary(filename, use_decoded=True)
+        else:
+            self.proto_analyzer.to_xml_file(filename=filename, decoders=self.decodings,
+                                            participants=self.project_manager.participants, write_bits=True)
 
     def show_differences(self, show_differences: bool):
         if show_differences:

src/urh/controller/MainController.py

@@ -367,7 +367,7 @@ def add_files(self, filepaths, group_id=0, enforce_sample_rate=None):
                 self.add_signalfile(filename, group_id, enforce_sample_rate=enforce_sample_rate)
             elif filename.endswith(".coco"):
                 self.add_signalfile(filename, group_id, enforce_sample_rate=enforce_sample_rate)
-            elif filename.endswith(".proto") or filename.endswith(".proto.xml"):
+            elif filename.endswith(".proto") or filename.endswith(".proto.xml") or filename.endswith(".bin"):
                 self.add_protocol_file(filename)
             elif filename.endswith(".wav"):
                 try:

src/urh/signalprocessing/ProtocolAnalyzer.py

@@ -2,7 +2,6 @@
 import copy
 import sys
 import xml.etree.ElementTree as ET
-from collections import defaultdict
 from xml.dom import minidom
 
 import numpy as np
@@ -41,7 +40,7 @@ class ProtocolAnalyzer(object):
     This class offers several methods for protocol analysis.
     """
 
-    def __init__(self, signal: Signal, filename=None):
+    def __init__(self, signal: Signal or None, filename=None):
         self.messages = []  # type: list[Message]
         self.signal = signal
         if filename is None:
@@ -546,6 +545,18 @@ def add_new_message_type(self, labels):
                     MessageType(name=name + str(i), iterable=[copy.deepcopy(lbl) for lbl in labels]))
                 break
 
+    def to_binary(self, filename: str, use_decoded: bool):
+        with open(filename, "wb") as f:
+            for msg in self.messages:
+                bits = msg.decoded_bits if use_decoded else msg.plain_bits
+                aggregated = urh_util.aggregate_bits(bits, size=8)
+                f.write(bytes(aggregated))
+
+    def from_binary(self, filename: str):
+        aggregated = np.fromfile(filename, dtype=np.uint8)
+        unaggregated = [int(b) for n in aggregated for b in "{0:08b}".format(n)]
+        self.messages.append(Message(unaggregated, 0, self.default_message_type))
+
     def to_xml_tag(self, decodings, participants, tag_name="protocol",
                    include_message_type=False, write_bits=False, messages=None, modulators=None) -> ET.Element:
         root = ET.Element(tag_name)
@@ -766,4 +777,4 @@ def parse_line(line: str):
                 bit_str = [lookup[bits[i].lower()] for i in range(0, len(bits))]
                 protocol.messages.append(Message.from_plain_bits_str("".join(bit_str), pause=pause))
 
-        return protocol
+        return protocol

src/urh/util/FileOperator.py

@@ -39,15 +39,16 @@ def get_open_dialog(directory_mode=False, parent=None, name_filter="full") -> QF
                           "Complex16 signed (*.complex16s);;" \
                           "Wave (*.wav);;" \
                           "Protocols (*.proto.xml *.proto);;" \
+                          "Binary Protocols (*.bin);;" \
                           "Fuzzprofiles (*.fuzz.xml *.fuzz);;" \
                           "Simulator (*.sim.xml *.sim)" \
                           "Plain bits (*.txt);;" \
                           "Tar Archives (*.tar *.tar.gz *.tar.bz2);;" \
                           "Zip Archives (*.zip)"
         elif name_filter == "proto":
-            name_filter = "Protocols (*.proto.xml *.proto);;"
+            name_filter = "Protocols (*.proto.xml *.proto);; Binary Protocols (*.bin)"
         elif name_filter == "fuzz":
-            name_filter = "Fuzzprofiles (*.fuzz.xml *.fuzz);;"
+            name_filter = "Fuzzprofiles (*.fuzz.xml *.fuzz)"
         elif name_filter == "simulator":
             name_filter = "Simulator (*.sim.xml *.sim)"
 
@@ -111,7 +112,7 @@ def get_save_file_name(initial_name: str, wav_only=False, caption="Save signal")
     elif caption == "Export spectrogram":
         name_filter = "Frequency Time (*.ft);;Frequency Time Amplitude (*.fta)"
     else:
-        name_filter = "Protocols (*.proto.xml *.proto);;All files (*)"
+        name_filter = "Protocols (*.proto.xml *.proto);;Binary Protocol (*.bin);;All files (*)"
 
     filename = None
     dialog = QFileDialog()

tests/test_protocol_analyzer.py

@@ -1,6 +1,9 @@
+import os
+import tempfile
 import unittest
 
 from tests.utils_testing import get_path_for_data_file
+from urh.signalprocessing.Message import Message
 from urh.signalprocessing.ProtocolAnalyzer import ProtocolAnalyzer
 from urh.signalprocessing.Signal import Signal
 
@@ -50,3 +53,15 @@ def test_get_rssi_of_message(self):
         self.assertGreater(messages[1].rssi, messages[2].rssi)
         self.assertLess(messages[2].rssi, messages[3].rssi)
         self.assertLess(messages[-2].rssi, messages[-1].rssi)
+
+    def test_binary_format(self):
+        pa = ProtocolAnalyzer(None)
+        pa.messages.append(Message([1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1], 0, pa.default_message_type))
+        pa.messages.append(Message([1, 1, 1, 0, 1], 0, pa.default_message_type))
+
+        filename = os.path.join(tempfile.gettempdir(), "test_proto.bin")
+        pa.to_binary(filename, use_decoded=True)
+
+        pa.from_binary(filename)
+        self.assertEqual(len(pa.messages), 3)
+        self.assertEqual(pa.plain_bits_str[2], "111000111001101111101000")