From 21499d9ae9edccd18c64bf6b15f248575c3da95b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 14:22:04 +0200 Subject: util: added global sliding memory manager pack: now using the global sliding memory manager. The current implementation uses assumes that packs are small enough to fit into memory right away, where the window size will be about 1 GB, as it never calls use_window() to assure the required offset actually exists. It will need to change to set the window appropriately. --- git/util.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index a31e5865..0c018447 100644 --- a/git/util.py +++ b/git/util.py @@ -15,6 +15,12 @@ import time import stat import shutil import tempfile +from smmap import ( + StaticWindowMapManager, + SlidingWindowMapBuffer + ) + + __all__ = ( "stream_copy", "join_path", "to_native_path_windows", "to_native_path_linux", "join_path_native", "Stats", "IndexFileSHA1Writer", "Iterable", "IterableList", @@ -64,6 +70,10 @@ except ImportError: # will be handled in the main thread pool = ThreadPool(0) +# initialize our global memory manager instance +# Use it to free cached (and unused) resources. +mman = StaticWindowMapManager() + #} END globals -- cgit v1.2.3 From f0c05ea8da7932961af162bb30231640b89e40bc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 18:13:30 +0200 Subject: util: pick the type of memory manager based on the python version, to have optimal results in all cases (at least the ones I can test) pack: now works properly with a sliding memory manager test_packedodb_pure: fixed very memory hungry implementation by using an iterator. This will of course reduce the measured performance a bit, but 750MB of memory is just a little bit too much for an ordinary test. Maybe it would be alright to just reduce the number of items ... but performance isn't a strength of python after all --- git/util.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index 0c018447..0e7e4cba 100644 --- a/git/util.py +++ b/git/util.py @@ -17,6 +17,7 @@ import shutil import tempfile from smmap import ( StaticWindowMapManager, + SlidingWindowMapManager, SlidingWindowMapBuffer ) @@ -72,7 +73,11 @@ pool = ThreadPool(0) # initialize our global memory manager instance # Use it to free cached (and unused) resources. -mman = StaticWindowMapManager() +if sys.version_info[1] < 6: + mman = StaticWindowMapManager() +else: + mman = SlidingWindowMapManager() +#END handle mman #} END globals -- cgit v1.2.3