From fabeffe39e70e1c371bcce158b3b8b0dd029e7a6 Mon Sep 17 00:00:00 2001
From: horpto <__Singleton__@hackerdom.ru>
Date: Fri, 18 Jan 2019 10:09:49 +0500
Subject: [PATCH] Reduce memory consumption of summarizer (#2298)

* reduce memory consumption

* fix build

* add deleting null-weighted edge

* iterate over bm25 weights

* fix build

* fix wrong example

* add method iter_graph in Graph

* add index argument in SyntacticUnit

* fix logging messages

* refactor graph - remove unnecessary parts

* Add test and fix typos

* fix built (flake8)

* add printing of documents number
---
 gensim/summarization/bm25.py              |  89 ++++++++-
 gensim/summarization/graph.py             | 217 +++++-----------------
 gensim/summarization/keywords.py          |   4 +-
 gensim/summarization/pagerank_weighted.py |  37 ++--
 gensim/summarization/summarizer.py        |  32 ++--
 gensim/summarization/syntactic_unit.py    |   4 +-
 gensim/summarization/textcleaner.py       |   3 +-
 gensim/test/test_summarization.py         |  95 +++++++++-
 8 files changed, 280 insertions(+), 201 deletions(-)

diff --git a/gensim/summarization/bm25.py b/gensim/summarization/bm25.py
index 18af369be7..fa8552c35e 100644
--- a/gensim/summarization/bm25.py
+++ b/gensim/summarization/bm25.py
@@ -162,6 +162,48 @@ def get_scores(self, document):
         scores = [self.get_score(document, index) for index in range(self.corpus_size)]
         return scores
 
+    def get_scores_bow(self, document):
+        """Computes and returns BM25 scores of given `document` in relation to
+        every item in corpus.
+
+        Parameters
+        ----------
+        document : list of str
+            Document to be scored.
+
+        Returns
+        -------
+        list of float
+            BM25 scores.
+
+        """
+        scores = []
+        for index in range(self.corpus_size):
+            score = self.get_score(document, index)
+            if score > 0:
+                scores.append((index, score))
+        return scores
+
+
+def _get_scores_bow(bm25, document):
+    """Helper function for retrieving bm25 scores of given `document` in parallel
+    in relation to every item in corpus.
+
+    Parameters
+    ----------
+    bm25 : BM25 object
+        BM25 object fitted on the corpus where documents are retrieved.
+    document : list of str
+        Document to be scored.
+
+    Returns
+    -------
+    list of (index, float)
+        BM25 scores in a bag of weights format.
+
+    """
+    return bm25.get_scores_bow(document)
+
 
 def _get_scores(bm25, document):
     """Helper function for retrieving bm25 scores of given `document` in parallel
@@ -183,6 +225,52 @@ def _get_scores(bm25, document):
     return bm25.get_scores(document)
 
 
+def iter_bm25_bow(corpus, n_jobs=1):
+    """Yield BM25 scores (weights) of documents in corpus.
+    Each document has to be weighted with every document in given corpus.
+
+    Parameters
+    ----------
+    corpus : list of list of str
+        Corpus of documents.
+    n_jobs : int
+        The number of processes to use for computing bm25.
+
+    Yields
+    -------
+    list of (index, float)
+        BM25 scores in bag of weights format.
+
+    Examples
+    --------
+    .. sourcecode:: pycon
+
+        >>> from gensim.summarization.bm25 import iter_bm25_weights
+        >>> corpus = [
+        ...     ["black", "cat", "white", "cat"],
+        ...     ["cat", "outer", "space"],
+        ...     ["wag", "dog"]
+        ... ]
+        >>> result = iter_bm25_weights(corpus, n_jobs=-1)
+
+    """
+    bm25 = BM25(corpus)
+
+    n_processes = effective_n_jobs(n_jobs)
+    if n_processes == 1:
+        for doc in corpus:
+            yield bm25.get_scores_bow(doc)
+        return
+
+    get_score = partial(_get_scores_bow, bm25)
+    pool = Pool(n_processes)
+
+    for bow in pool.imap(get_score, corpus):
+        yield bow
+    pool.close()
+    pool.join()
+
+
 def get_bm25_weights(corpus, n_jobs=1):
     """Returns BM25 scores (weights) of documents in corpus.
     Each document has to be weighted with every document in given corpus.
@@ -224,5 +312,4 @@ def get_bm25_weights(corpus, n_jobs=1):
     weights = pool.map(get_score, corpus)
     pool.close()
     pool.join()
-
     return weights
diff --git a/gensim/summarization/graph.py b/gensim/summarization/graph.py
index 12a43d06ce..9622ef7c7f 100644
--- a/gensim/summarization/graph.py
+++ b/gensim/summarization/graph.py
@@ -48,6 +48,11 @@ class IGraph(object):
     """
     __metaclass__ = ABCMeta
 
+    @abstractmethod
+    def __len__(self):
+        """Returns number of nodes in graph"""
+        pass
+
     @abstractmethod
     def nodes(self):
         """Returns all nodes of graph.
@@ -107,7 +112,7 @@ def has_node(self, node):
         pass
 
     @abstractmethod
-    def add_node(self, node, attrs=None):
+    def add_node(self, node):
         """Adds given node to the graph.
 
         Note
@@ -119,14 +124,12 @@ def add_node(self, node, attrs=None):
         ----------
         node : hashable
             Given node
-        attrs : list, optional
-            Node attributes specified as (attribute, value)
 
         """
         pass
 
     @abstractmethod
-    def add_edge(self, edge, wt=1, label='', attrs=None):
+    def add_edge(self, edge, wt=1):
         """Adds an edge to the graph connecting two nodes. An edge, here,
         is a tuple of two nodes.
 
@@ -136,10 +139,6 @@ def add_edge(self, edge, wt=1, label='', attrs=None):
             Given edge.
         wt : float, optional
             Weight of new edge.
-        label : str, optional
-            Edge label.
-        attrs : list, optional
-            Node attributes specified as (attribute, value)
 
         """
         pass
@@ -197,38 +196,24 @@ class Graph(IGraph):
 
     Attributes
     ----------
-    Graph.WEIGHT_ATTRIBUTE_NAME : str
-        Name of weight attribute in graph.
     Graph.DEFAULT_WEIGHT : float
         Weight set by default.
-    Graph.LABEL_ATTRIBUTE_NAME : str
-        Default name of attribute. Not used.
-    Graph.DEFAULT_LABEL : str
-        Label set by default. Not used.
 
     """
 
-    WEIGHT_ATTRIBUTE_NAME = "weight"
     DEFAULT_WEIGHT = 0
 
-    LABEL_ATTRIBUTE_NAME = "label"
-    DEFAULT_LABEL = ""
-
     def __init__(self):
         """Initializes object."""
-
-        # Metadata about edges
-        # Mapping: Edge -> Dict mapping, lablel-> str, wt->num
-        self.edge_properties = {}
-        # Key value pairs: (Edge -> Attributes)
-        self.edge_attr = {}
-
-        # Metadata about nodes
-        # Pairing: Node -> Attributes
-        self.node_attr = {}
-        # Pairing: Node -> Neighbors
+        # Pairing and metadata about edges
+        # Mapping: Node->
+        #    Dict mapping of Neighbor -> weight
         self.node_neighbors = {}
 
+    def __len__(self):
+        """Returns number of nodes in graph"""
+        return len(self.node_neighbors)
+
     def has_edge(self, edge):
         """Returns whether an edge exists.
 
@@ -244,7 +229,10 @@ def has_edge(self, edge):
 
         """
         u, v = edge
-        return (u, v) in self.edge_properties and (v, u) in self.edge_properties
+        return (u in self.node_neighbors
+            and v in self.node_neighbors
+            and v in self.node_neighbors[u]
+            and u in self.node_neighbors[v])
 
     def edge_weight(self, edge):
         """Returns weight of given edge.
@@ -260,7 +248,8 @@ def edge_weight(self, edge):
             Edge weight.
 
         """
-        return self.get_edge_properties(edge).setdefault(self.WEIGHT_ATTRIBUTE_NAME, self.DEFAULT_WEIGHT)
+        u, v = edge
+        return self.node_neighbors.get(u, {}).get(v, self.DEFAULT_WEIGHT)
 
     def neighbors(self, node):
         """Returns all nodes that are directly accessible from given node.
@@ -276,7 +265,7 @@ def neighbors(self, node):
             Nodes directly accessible from given `node`.
 
         """
-        return self.node_neighbors[node]
+        return list(self.node_neighbors[node])
 
     def has_node(self, node):
         """Returns whether the requested node exists.
@@ -294,7 +283,7 @@ def has_node(self, node):
         """
         return node in self.node_neighbors
 
-    def add_edge(self, edge, wt=1, label='', attrs=None):
+    def add_edge(self, edge, wt=1):
         """Adds an edge to the graph connecting two nodes.
 
         Parameters
@@ -303,10 +292,6 @@ def add_edge(self, edge, wt=1, label='', attrs=None):
             Given edge.
         wt : float, optional
             Weight of new edge.
-        label : str, optional
-            Edge label.
-        attrs : list, optional
-            Node attributes specified as (attribute, value).
 
         Raises
         ------
@@ -314,20 +299,20 @@ def add_edge(self, edge, wt=1, label='', attrs=None):
             If `edge` already exists in graph.
 
         """
-        if attrs is None:
-            attrs = []
+        if wt == 0.0:
+            # empty edge is similar to no edge at all or removing it
+            if self.has_edge(edge):
+                self.del_edge(edge)
+            return
         u, v = edge
         if v not in self.node_neighbors[u] and u not in self.node_neighbors[v]:
-            self.node_neighbors[u].append(v)
+            self.node_neighbors[u][v] = wt
             if u != v:
-                self.node_neighbors[v].append(u)
-
-            self.add_edge_attributes((u, v), attrs)
-            self.set_edge_properties((u, v), label=label, weight=wt)
+                self.node_neighbors[v][u] = wt
         else:
             raise ValueError("Edge (%s, %s) already in graph" % (u, v))
 
-    def add_node(self, node, attrs=None):
+    def add_node(self, node):
         """Adds given node to the graph.
 
         Note
@@ -340,8 +325,6 @@ def add_node(self, node, attrs=None):
         ----------
         node : hashable
             Given node.
-        attrs : list of (hashable, hashable), optional
-            Node attributes specified as (attribute, value)
 
         Raises
         ------
@@ -349,14 +332,11 @@ def add_node(self, node, attrs=None):
             If `node` already exists in graph.
 
         """
-        if attrs is None:
-            attrs = []
-        if node not in self.node_neighbors:
-            self.node_neighbors[node] = []
-            self.node_attr[node] = attrs
-        else:
+        if node in self.node_neighbors:
             raise ValueError("Node %s already in graph" % node)
 
+        self.node_neighbors[node] = {}
+
     def nodes(self):
         """Returns all nodes of the graph.
 
@@ -366,7 +346,7 @@ def nodes(self):
             Nodes of graph.
 
         """
-        return list(self.node_neighbors.keys())
+        return list(self.node_neighbors)
 
     def edges(self):
         """Returns all edges of the graph.
@@ -377,7 +357,20 @@ def edges(self):
             Edges of graph.
 
         """
-        return [a for a in self.edge_properties.keys()]
+        return list(self.iter_edges())
+
+    def iter_edges(self):
+        """Returns iterator of all edges of the graph.
+
+        Yields
+        -------
+        (hashable, hashable)
+            Edges of graph.
+
+        """
+        for u in self.node_neighbors:
+            for v in self.node_neighbors[u]:
+                yield (u, v)
 
     def del_node(self, node):
         """Removes given node and its edges from the graph.
@@ -388,98 +381,10 @@ def del_node(self, node):
             Given node.
 
         """
-        for each in list(self.neighbors(node)):
+        for each in self.neighbors(node):
             if each != node:
                 self.del_edge((each, node))
         del self.node_neighbors[node]
-        del self.node_attr[node]
-
-    def get_edge_properties(self, edge):
-        """Returns properties of given given edge. If edge doesn't exist
-        empty dictionary will be returned.
-
-        Parameters
-        ----------
-        edge : (hashable, hashable)
-            Given edge.
-
-        Returns
-        -------
-        dict
-            Properties of graph.
-
-        """
-        return self.edge_properties.setdefault(edge, {})
-
-    def add_edge_attributes(self, edge, attrs):
-        """Adds attributes `attrs` to given edge, order of nodes in edge doesn't matter.
-
-        Parameters
-        ----------
-        edge : (hashable, hashable)
-            Given edge.
-        attrs : list
-            Provided attributes to add.
-
-        """
-        for attr in attrs:
-            self.add_edge_attribute(edge, attr)
-
-    def add_edge_attribute(self, edge, attr):
-        """Adds attribute `attr` to given edge, order of nodes in edge doesn't matter.
-
-        Parameters
-        ----------
-        edge : (hashable, hashable)
-            Given edge.
-
-        attr : object
-            Provided attribute to add.
-
-        """
-        self.edge_attr[edge] = self.edge_attributes(edge) + [attr]
-
-        if edge[0] != edge[1]:
-            self.edge_attr[(edge[1], edge[0])] = self.edge_attributes((edge[1], edge[0])) + [attr]
-
-    def edge_attributes(self, edge):
-        """Returns attributes of given edge.
-
-        Note
-        ----
-        In case of non existing edge returns empty list.
-
-        Parameters
-        ----------
-        edge : (hashable, hashable)
-            Given edge.
-
-        Returns
-        -------
-        list
-            Attributes of given edge.
-
-        """
-        try:
-            return self.edge_attr[edge]
-        except KeyError:
-            return []
-
-    def set_edge_properties(self, edge, **properties):
-        """Adds `properties` to given edge, order of nodes in edge doesn't matter.
-
-        Parameters
-        ----------
-        edge : (hashable, hashable)
-            Given edge.
-
-        properties : dict
-            Properties to add.
-
-        """
-        self.edge_properties.setdefault(edge, {}).update(properties)
-        if edge[0] != edge[1]:
-            self.edge_properties.setdefault((edge[1], edge[0]), {}).update(properties)
 
     def del_edge(self, edge):
         """Removes given edges from the graph.
@@ -491,26 +396,6 @@ def del_edge(self, edge):
 
         """
         u, v = edge
-        self.node_neighbors[u].remove(v)
-        self.del_edge_labeling((u, v))
+        del self.node_neighbors[u][v]
         if u != v:
-            self.node_neighbors[v].remove(u)
-            self.del_edge_labeling((v, u))
-
-    def del_edge_labeling(self, edge):
-        """Removes attributes and properties of given edge.
-
-        Parameters
-        ----------
-        edge : (hashable, hashable)
-            Given edge.
-
-        """
-        keys = [edge, edge[::-1]]
-
-        for key in keys:
-            for mapping in [self.edge_properties, self.edge_attr]:
-                try:
-                    del mapping[key]
-                except KeyError:
-                    pass
+            del self.node_neighbors[v][u]
diff --git a/gensim/summarization/keywords.py b/gensim/summarization/keywords.py
index 80f293517c..db7c8a0dc7 100644
--- a/gensim/summarization/keywords.py
+++ b/gensim/summarization/keywords.py
@@ -97,7 +97,7 @@ def _get_words_for_graph(tokens, pos_filter=None):
     for word, unit in iteritems(tokens):
         if exclude_filters and unit.tag in exclude_filters:
             continue
-        if (include_filters and unit.tag in include_filters) or not include_filters or not unit.tag:
+        if not include_filters or not unit.tag or unit.tag in include_filters:
             result.append(unit.token)
     return result
 
@@ -511,7 +511,7 @@ def keywords(text, ratio=0.2, words=None, split=False, scores=False, pos_filter=
 
     _remove_unreachable_nodes(graph)
 
-    if not graph.edges():
+    if not any(True for _ in graph.iter_edges()):
         return _format_results([], [], split, scores)
 
     # Ranks the tokens using the PageRank algorithm. Returns dict of lemma -> score
diff --git a/gensim/summarization/pagerank_weighted.py b/gensim/summarization/pagerank_weighted.py
index f961d6b729..50562eb032 100644
--- a/gensim/summarization/pagerank_weighted.py
+++ b/gensim/summarization/pagerank_weighted.py
@@ -43,6 +43,8 @@
 from scipy.sparse.linalg import eigs
 from six.moves import range
 
+from gensim.utils import deprecated
+
 
 def pagerank_weighted(graph, damping=0.85):
     """Get dictionary of `graph` nodes and its ranks.
@@ -60,10 +62,12 @@ def pagerank_weighted(graph, damping=0.85):
         Nodes of `graph` as keys, its ranks as values.
 
     """
-    adjacency_matrix = build_adjacency_matrix(graph)
-    probability_matrix = build_probability_matrix(graph)
+    coeff_adjacency_matrix = build_adjacency_matrix(graph, coeff=damping)
+    probabilities = (1 - damping) / float(len(graph))
 
-    pagerank_matrix = damping * adjacency_matrix.todense() + (1 - damping) * probability_matrix
+    pagerank_matrix = coeff_adjacency_matrix.toarray()
+    # trying to minimize memory allocations
+    pagerank_matrix += probabilities
 
     vec = principal_eigenvector(pagerank_matrix.T)
 
@@ -71,13 +75,15 @@ def pagerank_weighted(graph, damping=0.85):
     return process_results(graph, vec.real)
 
 
-def build_adjacency_matrix(graph):
+def build_adjacency_matrix(graph, coeff=1):
     """Get matrix representation of given `graph`.
 
     Parameters
     ----------
     graph : :class:`~gensim.summarization.graph.Graph`
         Given graph.
+    coeff : float
+        Matrix values coefficient, optonal.
 
     Returns
     -------
@@ -89,22 +95,25 @@ def build_adjacency_matrix(graph):
     col = []
     data = []
     nodes = graph.nodes()
+    nodes2id = {v: i for i, v in enumerate(nodes)}
     length = len(nodes)
 
     for i in range(length):
         current_node = nodes[i]
-        neighbors_sum = sum(graph.edge_weight((current_node, neighbor)) for neighbor in graph.neighbors(current_node))
-        for j in range(length):
-            edge_weight = float(graph.edge_weight((current_node, nodes[j])))
-            if i != j and edge_weight != 0.0:
+        neighbors = graph.neighbors(current_node)
+        neighbors_sum = sum(graph.edge_weight((current_node, neighbor)) for neighbor in neighbors)
+        for neighbor in neighbors:
+            edge_weight = float(graph.edge_weight((current_node, neighbor)))
+            if edge_weight != 0.0:
                 row.append(i)
-                col.append(j)
-                data.append(edge_weight / neighbors_sum)
+                col.append(nodes2id[neighbor])
+                data.append(coeff * edge_weight / neighbors_sum)
 
     return csr_matrix((data, (row, col)), shape=(length, length))
 
 
-def build_probability_matrix(graph):
+@deprecated("Function will be removed in 4.0.0")
+def build_probability_matrix(graph, coeff=1.0):
     """Get square matrix of shape (n, n), where n is number of nodes of the
     given `graph`.
 
@@ -112,6 +121,8 @@ def build_probability_matrix(graph):
     ----------
     graph : :class:`~gensim.summarization.graph.Graph`
         Given graph.
+    coeff : float
+        Matrix values coefficient, optonal.
 
     Returns
     -------
@@ -119,10 +130,10 @@ def build_probability_matrix(graph):
         Eigenvector of matrix `a`, n is number of nodes of `graph`.
 
     """
-    dimension = len(graph.nodes())
+    dimension = len(graph)
     matrix = empty_matrix((dimension, dimension))
 
-    probability = 1.0 / float(dimension)
+    probability = coeff / float(dimension)
     matrix.fill(probability)
 
     return matrix
diff --git a/gensim/summarization/summarizer.py b/gensim/summarization/summarizer.py
index 3c81d99d5b..73a2fba26f 100644
--- a/gensim/summarization/summarizer.py
+++ b/gensim/summarization/summarizer.py
@@ -58,7 +58,7 @@
 from gensim.summarization.textcleaner import clean_text_by_sentences as _clean_text_by_sentences
 from gensim.summarization.commons import build_graph as _build_graph
 from gensim.summarization.commons import remove_unreachable_nodes as _remove_unreachable_nodes
-from gensim.summarization.bm25 import get_bm25_weights as _bm25_weights
+from gensim.summarization.bm25 import iter_bm25_bow as _bm25_weights
 from gensim.corpora import Dictionary
 from math import log10 as _log10
 from six.moves import range
@@ -84,25 +84,22 @@ def _set_graph_edge_weights(graph):
     documents = graph.nodes()
     weights = _bm25_weights(documents)
 
-    for i in range(len(documents)):
-        for j in range(len(documents)):
-            if i == j or weights[i][j] < WEIGHT_THRESHOLD:
-                continue
+    for i, doc_bow in enumerate(weights):
+        if i % 1000 == 0 and i > 0:
+            logger.info('PROGRESS: processing %s/%s doc (%s non zero elements)', i, len(documents), len(doc_bow))
 
-            sentence_1 = documents[i]
-            sentence_2 = documents[j]
+        for j, weight in doc_bow:
+            if i == j or weight < WEIGHT_THRESHOLD:
+                continue
 
-            edge_1 = (sentence_1, sentence_2)
-            edge_2 = (sentence_2, sentence_1)
+            edge = (documents[i], documents[j])
 
-            if not graph.has_edge(edge_1):
-                graph.add_edge(edge_1, weights[i][j])
-            if not graph.has_edge(edge_2):
-                graph.add_edge(edge_2, weights[j][i])
+            if not graph.has_edge(edge):
+                graph.add_edge(edge, weight)
 
     # Handles the case in which all similarities are zero.
     # The resultant summary will consist of random sentences.
-    if all(graph.edge_weight(edge) == 0 for edge in graph.edges()):
+    if all(graph.edge_weight(edge) == 0 for edge in graph.iter_edges()):
         _create_valid_graph(graph)
 
 
@@ -358,8 +355,13 @@ def summarize_corpus(corpus, ratio=0.2):
     if len(corpus) < INPUT_MIN_LENGTH:
         logger.warning("Input corpus is expected to have at least %d documents.", INPUT_MIN_LENGTH)
 
+    logger.info('Building graph')
     graph = _build_graph(hashable_corpus)
+
+    logger.info('Filling graph')
     _set_graph_edge_weights(graph)
+
+    logger.info('Removing unreachable nodes of graph')
     _remove_unreachable_nodes(graph)
 
     # Cannot calculate eigenvectors if number of unique documents in corpus < 3.
@@ -368,8 +370,10 @@ def summarize_corpus(corpus, ratio=0.2):
         logger.warning("Please add more sentences to the text. The number of reachable nodes is below 3")
         return []
 
+    logger.info('Pagerank graph')
     pagerank_scores = _pagerank(graph)
 
+    logger.info('Sorting pagerank scores')
     hashable_corpus.sort(key=lambda doc: pagerank_scores.get(doc, 0), reverse=True)
 
     return [list(doc) for doc in hashable_corpus[:int(len(corpus) * ratio)]]
diff --git a/gensim/summarization/syntactic_unit.py b/gensim/summarization/syntactic_unit.py
index 335ee6a212..2926e4fe1b 100644
--- a/gensim/summarization/syntactic_unit.py
+++ b/gensim/summarization/syntactic_unit.py
@@ -27,7 +27,7 @@ class SyntacticUnit(object):
 
     """
 
-    def __init__(self, text, token=None, tag=None):
+    def __init__(self, text, token=None, tag=None, index=-1):
         """
 
         Parameters
@@ -43,7 +43,7 @@ def __init__(self, text, token=None, tag=None):
         self.text = text
         self.token = token
         self.tag = tag[:2] if tag else None  # Just first two letters of tag
-        self.index = -1
+        self.index = index
         self.score = -1
 
     def __str__(self):
diff --git a/gensim/summarization/textcleaner.py b/gensim/summarization/textcleaner.py
index 9223b72f10..ba51b7691a 100644
--- a/gensim/summarization/textcleaner.py
+++ b/gensim/summarization/textcleaner.py
@@ -205,8 +205,7 @@ def merge_syntactic_units(original_units, filtered_units, tags=None):
         text = original_units[i]
         token = filtered_units[i]
         tag = tags[i][1] if tags else None
-        sentence = SyntacticUnit(text, token, tag)
-        sentence.index = i
+        sentence = SyntacticUnit(text, token, tag, i)
 
         units.append(sentence)
 
diff --git a/gensim/test/test_summarization.py b/gensim/test/test_summarization.py
index 7024385bba..81a562a9d8 100644
--- a/gensim/test/test_summarization.py
+++ b/gensim/test/test_summarization.py
@@ -20,9 +20,10 @@
 from gensim.corpora import Dictionary
 from gensim.summarization import summarize, summarize_corpus, keywords, mz_keywords
 from gensim.summarization.commons import remove_unreachable_nodes, build_graph
+from gensim.summarization.graph import Graph
 
 
-class TestCommons(unittest.TestCase):
+class TestGraph(unittest.TestCase):
 
     def _build_graph(self):
         graph = build_graph(['a', 'b', 'c', 'd'])
@@ -31,6 +32,17 @@ def _build_graph(self):
         graph.add_edge(('c', 'a'))
         return graph
 
+    def test_build_graph(self):
+        graph = self._build_graph()
+
+        self.assertEqual(sorted(graph.nodes()), ['a', 'b', 'c', 'd'])
+        self.assertTrue(graph.has_edge(('a', 'b')))
+        self.assertTrue(graph.has_edge(('b', 'c')))
+        self.assertTrue(graph.has_edge(('c', 'a')))
+
+        graph = build_graph([])
+        self.assertEqual(graph.nodes(), [])
+
     def test_remove_unreachable_nodes(self):
         graph = self._build_graph()
         self.assertTrue(graph.has_node('d'))
@@ -44,6 +56,87 @@ def test_remove_unreachable_nodes(self):
         remove_unreachable_nodes(graph)
         self.assertFalse(graph.has_node('d'))
 
+    def test_graph_nodes(self):
+        graph = Graph()
+
+        graph.add_node('a')
+        graph.add_node(1)
+        graph.add_node('b')
+        graph.add_node('qwe')
+
+        self.assertTrue(graph.has_node('a'))
+        self.assertTrue(graph.has_node('b'))
+        self.assertTrue(graph.has_node('qwe'))
+        self.assertTrue(graph.has_node(1))
+        self.assertFalse(graph.has_node(2))
+
+        graph.del_node(1)
+        self.assertEqual(sorted(graph.nodes()), ['a', 'b', 'qwe'])
+
+    def test_graph_edges(self):
+        graph = Graph()
+        for node in ('a', 'b', 'c', 'd', 'e', 'foo', 'baz', 'qwe', 'rtyu'):
+            graph.add_node(node)
+
+        edges = [
+            (('a', 'b'), 3.0),
+            (('c', 'b'), 5.0),
+            (('d', 'e'), 0.5),
+            (('a', 'c'), 0.1),
+            (('foo', 'baz'), 0.11),
+            (('qwe', 'rtyu'), 0.0),
+        ]
+        for edge, weight in edges:
+            graph.add_edge(edge, weight)
+
+        # check on edge weight first to exclude situation when touching will create an edge
+        self.assertEqual(graph.edge_weight(('qwe', 'rtyu')), 0.0)
+        self.assertEqual(graph.edge_weight(('rtyu', 'qwe')), 0.0)
+        self.assertFalse(graph.has_edge(('qwe', 'rtyu')))
+        self.assertFalse(graph.has_edge(('rtyu', 'qwe')))
+
+        for (u, v), weight in edges:
+            if weight == 0:
+                continue
+            self.assertTrue(graph.has_edge((u, v)))
+            self.assertTrue(graph.has_edge((v, u)))
+
+        edges_list = [(u, v) for (u, v), w in edges if w]
+        edges_list.extend((v, u) for (u, v), w in edges if w)
+        edges_list.sort()
+
+        self.assertEqual(sorted(graph.iter_edges()), edges_list)
+
+        ret_edges = graph.edges()
+        ret_edges.sort()
+        self.assertEqual(ret_edges, edges_list)
+
+        for (u, v), weight in edges:
+            self.assertEqual(graph.edge_weight((u, v)), weight)
+            self.assertEqual(graph.edge_weight((v, u)), weight)
+
+        self.assertEqual(sorted(graph.neighbors('a')), ['b', 'c'])
+        self.assertEqual(sorted(graph.neighbors('b')), ['a', 'c'])
+        self.assertEqual(graph.neighbors('d'), ['e'])
+        self.assertEqual(graph.neighbors('e'), ['d'])
+        self.assertEqual(graph.neighbors('foo'), ['baz'])
+        self.assertEqual(graph.neighbors('baz'), ['foo'])
+        self.assertEqual(graph.neighbors('foo'), ['baz'])
+        self.assertEqual(graph.neighbors('qwe'), [])
+        self.assertEqual(graph.neighbors('rtyu'), [])
+
+        graph.del_edge(('a', 'b'))
+        self.assertFalse(graph.has_edge(('a', 'b')))
+        self.assertFalse(graph.has_edge(('b', 'a')))
+
+        graph.add_edge(('baz', 'foo'), 0)
+        self.assertFalse(graph.has_edge(('foo', 'baz')))
+        self.assertFalse(graph.has_edge(('baz', 'foo')))
+
+        graph.del_node('b')
+        self.assertFalse(graph.has_edge(('b', 'c')))
+        self.assertFalse(graph.has_edge(('c', 'b')))
+
 
 class TestSummarizationTest(unittest.TestCase):