From b72e2704022d889f116e49abf3e1e5d3e3192d3b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 6 Jun 2010 01:00:12 +0200 Subject: Improved pool design and started rough implementation, top down to learn while going. Tests will be written soon for verification, its still quite theoretical --- test/git/async/test_graph.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 test/git/async/test_graph.py (limited to 'test/git/async/test_graph.py') diff --git a/test/git/async/test_graph.py b/test/git/async/test_graph.py new file mode 100644 index 00000000..18d6997c --- /dev/null +++ b/test/git/async/test_graph.py @@ -0,0 +1,10 @@ +"""Channel testing""" +from test.testlib import * +from git.async.graph import * + +import time + +class TestGraph(TestBase): + + def test_base(self): + pass -- cgit v1.2.3 From ec28ad575ce1d7bb6a616ffc404f32bbb1af67b2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 6 Jun 2010 12:48:25 +0200 Subject: thread: adjusted worker thread not to provide an output queue anymore - this is handled by the task system graph: implemented it including test according to the pools requirements pool: implemented set_pool_size --- test/git/async/test_graph.py | 82 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) (limited to 'test/git/async/test_graph.py') diff --git a/test/git/async/test_graph.py b/test/git/async/test_graph.py index 18d6997c..400e92cd 100644 --- a/test/git/async/test_graph.py +++ b/test/git/async/test_graph.py @@ -7,4 +7,84 @@ import time class TestGraph(TestBase): def test_base(self): - pass + g = Graph() + nn = 10 + assert nn > 2, "need at least 3 nodes" + + # add unconnected nodes + for i in range(nn): + assert isinstance(g.add_node(Node()), Node) + # END add nodes + assert len(g.nodes) == nn + + # delete unconnected nodes + for n in g.nodes[:]: + g.del_node(n) + # END del nodes + + # add a chain of connected nodes + last = None + for i in range(nn): + n = g.add_node(Node()) + if last: + assert not last.out_nodes + assert not n.in_nodes + assert g.add_edge(last, n) is g + assert last.out_nodes[0] is n + assert n.in_nodes[0] is last + last = n + # END for each node to connect + + # try to connect a node with itself + self.failUnlessRaises(ValueError, g.add_edge, last, last) + + # try to create a cycle + self.failUnlessRaises(ValueError, g.add_edge, g.nodes[0], g.nodes[-1]) + self.failUnlessRaises(ValueError, g.add_edge, g.nodes[-1], g.nodes[0]) + + # we have undirected edges, readding the same edge, but the other way + # around does not change anything + n1, n2, n3 = g.nodes[0], g.nodes[1], g.nodes[2] + g.add_edge(n1, n2) # already connected + g.add_edge(n2, n1) # same thing + assert len(n1.out_nodes) == 1 + assert len(n1.in_nodes) == 0 + assert len(n2.in_nodes) == 1 + assert len(n2.out_nodes) == 1 + + # deleting a connected node clears its neighbour connections + assert n3.in_nodes[0] is n2 + g.del_node(n2) + assert len(g.nodes) == nn - 1 + assert len(n3.in_nodes) == 0 + assert len(n1.out_nodes) == 0 + + # check the history from the last node + last = g.nodes[-1] + class Visitor(object): + def __init__(self, origin): + self.origin_seen = False + self.origin = origin + self.num_seen = 0 + + def __call__(self, n): + if n is self.origin: + self.origin_seen = True + else: + assert not self.origin_seen, "should see origin last" + # END check origin + self.num_seen += 1 + return True + + def _assert(self, num_expected): + assert self.origin_seen + assert self.num_seen == num_expected + # END visitor helper + + end = g.nodes[-1] + visitor = Visitor(end) + g.visit_input_inclusive_depth_first(end, visitor) + + num_nodes_seen = nn - 2 # deleted second, which leaves first one disconnected + visitor._assert(num_nodes_seen) + -- cgit v1.2.3 From b3cde0ee162b8f0cb67da981311c8f9c16050a62 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 6 Jun 2010 18:13:21 +0200 Subject: First step of testing the pool - tasks have been separated into a new module including own tests, their design improved to prepare them for some specifics that would be needed for multiprocessing support --- test/git/async/test_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/git/async/test_graph.py') diff --git a/test/git/async/test_graph.py b/test/git/async/test_graph.py index 400e92cd..ca17d6e6 100644 --- a/test/git/async/test_graph.py +++ b/test/git/async/test_graph.py @@ -25,7 +25,7 @@ class TestGraph(TestBase): # add a chain of connected nodes last = None for i in range(nn): - n = g.add_node(Node()) + n = g.add_node(Node(i)) if last: assert not last.out_nodes assert not n.in_nodes -- cgit v1.2.3 From 1b27292936c81637f6b9a7141dafaad1126f268e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 6 Jun 2010 21:15:13 +0200 Subject: Plenty of fixes in the chunking routine, made possible by a serialized chunking test. Next up, actual async processing --- test/git/async/test_graph.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'test/git/async/test_graph.py') diff --git a/test/git/async/test_graph.py b/test/git/async/test_graph.py index ca17d6e6..1a153e2d 100644 --- a/test/git/async/test_graph.py +++ b/test/git/async/test_graph.py @@ -54,7 +54,8 @@ class TestGraph(TestBase): # deleting a connected node clears its neighbour connections assert n3.in_nodes[0] is n2 - g.del_node(n2) + assert g.del_node(n2) is g + assert g.del_node(n2) is g # multi-deletion okay assert len(g.nodes) == nn - 1 assert len(n3.in_nodes) == 0 assert len(n1.out_nodes) == 0 -- cgit v1.2.3 From 3323464f85b986cba23176271da92a478b33ab9c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 10 Jun 2010 00:24:49 +0200 Subject: messy first version of a properly working depth-first graph method, which allows the pool to work as expected. Many more tests need to be added, and there still is a problem with shutdown as sometimes it won't kill all threads, mainly because the process came up with worker threads started, which cannot be --- test/git/async/test_graph.py | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) (limited to 'test/git/async/test_graph.py') diff --git a/test/git/async/test_graph.py b/test/git/async/test_graph.py index 1a153e2d..d0e36159 100644 --- a/test/git/async/test_graph.py +++ b/test/git/async/test_graph.py @@ -61,31 +61,10 @@ class TestGraph(TestBase): assert len(n1.out_nodes) == 0 # check the history from the last node - last = g.nodes[-1] - class Visitor(object): - def __init__(self, origin): - self.origin_seen = False - self.origin = origin - self.num_seen = 0 - - def __call__(self, n): - if n is self.origin: - self.origin_seen = True - else: - assert not self.origin_seen, "should see origin last" - # END check origin - self.num_seen += 1 - return True - - def _assert(self, num_expected): - assert self.origin_seen - assert self.num_seen == num_expected - # END visitor helper - end = g.nodes[-1] - visitor = Visitor(end) - g.visit_input_inclusive_depth_first(end, visitor) - + dfirst_nodes = g.input_inclusive_dfirst_reversed(end) num_nodes_seen = nn - 2 # deleted second, which leaves first one disconnected - visitor._assert(num_nodes_seen) + assert len(dfirst_nodes) == num_nodes_seen + assert dfirst_nodes[-1] == end and dfirst_nodes[-2].id == end.id-1 + -- cgit v1.2.3 From fbe062bf6dacd3ad63dd827d898337fa542931ac Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 10 Jun 2010 23:55:50 +0200 Subject: Added dependency-task tests, and fixed plenty of ref-count related bugs, as well as concurrency issues. Now it works okay, but the thread-shutdown is still an issue, as it causes incorrect behaviour making the tests fail. Its good, as it hints at additional issues that need to be solved. There is just a little more left on the feature side, but its nearly there --- test/git/async/test_graph.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'test/git/async/test_graph.py') diff --git a/test/git/async/test_graph.py b/test/git/async/test_graph.py index d0e36159..7630226b 100644 --- a/test/git/async/test_graph.py +++ b/test/git/async/test_graph.py @@ -3,6 +3,7 @@ from test.testlib import * from git.async.graph import * import time +import sys class TestGraph(TestBase): @@ -19,7 +20,7 @@ class TestGraph(TestBase): # delete unconnected nodes for n in g.nodes[:]: - g.del_node(n) + g.remove_node(n) # END del nodes # add a chain of connected nodes @@ -54,8 +55,8 @@ class TestGraph(TestBase): # deleting a connected node clears its neighbour connections assert n3.in_nodes[0] is n2 - assert g.del_node(n2) is g - assert g.del_node(n2) is g # multi-deletion okay + assert g.remove_node(n2) is g + assert g.remove_node(n2) is g # multi-deletion okay assert len(g.nodes) == nn - 1 assert len(n3.in_nodes) == 0 assert len(n1.out_nodes) == 0 @@ -68,3 +69,12 @@ class TestGraph(TestBase): assert dfirst_nodes[-1] == end and dfirst_nodes[-2].id == end.id-1 + # test cleanup + # its at least kept by its graph + assert sys.getrefcount(end) > 3 + del(g) + del(n1); del(n2); del(n3) + del(dfirst_nodes) + del(last) + del(n) + assert sys.getrefcount(end) == 2 -- cgit v1.2.3