diff options
Diffstat (limited to 'git')
| -rw-r--r-- | git/cmd.py | 17 | ||||
| -rw-r--r-- | git/diff.py | 25 | ||||
| m--------- | git/ext/gitdb | 0 | ||||
| -rw-r--r-- | git/objects/commit.py | 6 | ||||
| -rw-r--r-- | git/remote.py | 65 | ||||
| -rw-r--r-- | git/repo/base.py | 6 | ||||
| -rw-r--r-- | git/test/fixtures/commit_invalid_data | 6 | ||||
| -rw-r--r-- | git/test/fixtures/diff_patch_unsafe_paths | 14 | ||||
| -rw-r--r-- | git/test/test_commit.py | 7 | ||||
| -rw-r--r-- | git/test/test_diff.py | 14 | ||||
| -rw-r--r-- | git/test/test_docs.py | 9 | ||||
| -rw-r--r-- | git/test/test_git.py | 1 | ||||
| -rw-r--r-- | git/util.py | 34 |
13 files changed, 154 insertions, 50 deletions
@@ -14,7 +14,6 @@ import errno import mmap from git.odict import OrderedDict - from contextlib import contextmanager import signal from subprocess import ( @@ -36,6 +35,7 @@ from .exc import ( from git.compat import ( string_types, defenc, + force_bytes, PY3, bchr, # just to satisfy flake8 on py3 @@ -287,7 +287,7 @@ class Git(LazyMixin): return # can be that nothing really exists anymore ... - if os is None: + if os is None or os.kill is None: return # try to kill it @@ -307,22 +307,27 @@ class Git(LazyMixin): def __getattr__(self, attr): return getattr(self.proc, attr) - def wait(self, stderr=None): + def wait(self, stderr=b''): """Wait for the process and return its status code. :param stderr: Previously read value of stderr, in case stderr is already closed. :warn: may deadlock if output or error pipes are used and not handled separately. :raise GitCommandError: if the return status is not 0""" + if stderr is None: + stderr = b'' + stderr = force_bytes(stderr) + status = self.proc.wait() def read_all_from_possibly_closed_stream(stream): try: - return stream.read() + return stderr + force_bytes(stream.read()) except ValueError: - return stderr or '' + return stderr or b'' if status != 0: errstr = read_all_from_possibly_closed_stream(self.proc.stderr) + log.debug('AutoInterrupt wait stderr: %r' % (errstr,)) raise GitCommandError(self.args, status, errstr) # END status handling return status @@ -609,7 +614,7 @@ class Git(LazyMixin): bufsize=-1, stdin=istream, stderr=PIPE, - stdout=with_stdout and PIPE or open(os.devnull, 'wb'), + stdout=PIPE if with_stdout else open(os.devnull, 'wb'), shell=self.USE_SHELL, close_fds=(os.name == 'posix'), # unsupported on windows universal_newlines=universal_newlines, diff --git a/git/diff.py b/git/diff.py index 44a65017..aeaa67d5 100644 --- a/git/diff.py +++ b/git/diff.py @@ -15,12 +15,23 @@ from git.compat import ( PY3 ) - __all__ = ('Diffable', 'DiffIndex', 'Diff', 'NULL_TREE') # Special object to compare against the empty tree in diffs NULL_TREE = object() +_octal_byte_re = re.compile(b'\\\\([0-9]{3})') + + +def _octal_repl(matchobj): + value = matchobj.group(1) + value = int(value, 8) + if PY3: + value = bytes(bytearray((value,))) + else: + value = chr(value) + return value + def decode_path(path, has_ab_prefix=True): if path == b'/dev/null': @@ -32,6 +43,8 @@ def decode_path(path, has_ab_prefix=True): .replace(b'\\"', b'"') .replace(b'\\\\', b'\\')) + path = _octal_byte_re.sub(_octal_repl, path) + if has_ab_prefix: assert path.startswith(b'a/') or path.startswith(b'b/') path = path[2:] @@ -337,7 +350,7 @@ class Diff(object): :note: This property is deprecated, please use ``renamed_file`` instead. """ return self.renamed_file - + @property def renamed_file(self): """:returns: True if the blob of our diff has been renamed @@ -391,15 +404,15 @@ class Diff(object): a_mode = old_mode or deleted_file_mode or (a_path and (b_mode or new_mode or new_file_mode)) b_mode = b_mode or new_mode or new_file_mode or (b_path and a_mode) index.append(Diff(repo, - a_path and a_path.decode(defenc), - b_path and b_path.decode(defenc), + a_path and a_path.decode(defenc, 'replace'), + b_path and b_path.decode(defenc, 'replace'), a_blob_id and a_blob_id.decode(defenc), b_blob_id and b_blob_id.decode(defenc), a_mode and a_mode.decode(defenc), b_mode and b_mode.decode(defenc), new_file, deleted_file, - rename_from and rename_from.decode(defenc), - rename_to and rename_to.decode(defenc), + rename_from and rename_from.decode(defenc, 'replace'), + rename_to and rename_to.decode(defenc, 'replace'), None)) previous_header = header diff --git a/git/ext/gitdb b/git/ext/gitdb -Subproject 2389b75280efb1a63e6ea578eae7f897fd4beb1 +Subproject d1996e04dbf4841b853b60c1365f0f5fd28d170 diff --git a/git/objects/commit.py b/git/objects/commit.py index dc722f97..9e434c92 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -501,14 +501,14 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): try: self.author, self.authored_date, self.author_tz_offset = \ - parse_actor_and_date(author_line.decode(self.encoding)) + parse_actor_and_date(author_line.decode(self.encoding, 'replace')) except UnicodeDecodeError: log.error("Failed to decode author line '%s' using encoding %s", author_line, self.encoding, exc_info=True) try: self.committer, self.committed_date, self.committer_tz_offset = \ - parse_actor_and_date(committer_line.decode(self.encoding)) + parse_actor_and_date(committer_line.decode(self.encoding, 'replace')) except UnicodeDecodeError: log.error("Failed to decode committer line '%s' using encoding %s", committer_line, self.encoding, exc_info=True) @@ -518,7 +518,7 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): # The end of our message stream is marked with a newline that we strip self.message = stream.read() try: - self.message = self.message.decode(self.encoding) + self.message = self.message.decode(self.encoding, 'replace') except UnicodeDecodeError: log.error("Failed to decode message '%s' using encoding %s", self.message, self.encoding, exc_info=True) # END exception handling diff --git a/git/remote.py b/git/remote.py index 5e9fe2c0..75a6875f 100644 --- a/git/remote.py +++ b/git/remote.py @@ -8,7 +8,6 @@ import re import os -from .exc import GitCommandError from .config import ( SectionConstraint, cp, @@ -24,7 +23,8 @@ from git.util import ( LazyMixin, Iterable, IterableList, - RemoteProgress + RemoteProgress, + CallableRemoteProgress ) from git.util import ( join_path, @@ -49,8 +49,8 @@ def add_progress(kwargs, git, progress): given, we do not request any progress :return: possibly altered kwargs""" if progress is not None: - v = git.version_info - if v[0] > 1 or v[1] > 7 or v[2] > 0 or v[3] > 3: + v = git.version_info[:2] + if v >= (1, 7): kwargs['progress'] = True # END handle --progress # END handle progress @@ -59,6 +59,23 @@ def add_progress(kwargs, git, progress): #} END utilities +def to_progress_instance(progress): + """Given the 'progress' return a suitable object derived from + RemoteProgress(). + """ + # new API only needs progress as a function + if callable(progress): + return CallableRemoteProgress(progress) + + # where None is passed create a parser that eats the progress + elif progress is None: + return RemoteProgress() + + # assume its the old API with an instance of RemoteProgress. + else: + return progress + + class PushInfo(object): """ @@ -186,7 +203,7 @@ class FetchInfo(object): NEW_TAG, NEW_HEAD, HEAD_UPTODATE, TAG_UPDATE, REJECTED, FORCED_UPDATE, \ FAST_FORWARD, ERROR = [1 << x for x in range(8)] - re_fetch_result = re.compile("^\s*(.) (\[?[\w\s\.$@]+\]?)\s+(.+) -> ([/\w_\+\.\-$@#]+)( \(.*\)?$)?") + re_fetch_result = re.compile("^\s*(.) (\[?[\w\s\.$@]+\]?)\s+(.+) -> ([/\w_\+\.\-$@#()]+)( \(.*\)?$)?") _flag_map = {'!': ERROR, '+': FORCED_UPDATE, @@ -585,6 +602,8 @@ class Remote(LazyMixin, Iterable): return self def _get_fetch_info_from_stderr(self, proc, progress): + progress = to_progress_instance(progress) + # skip first line as it is some remote info we are not interested in output = IterableList('name') @@ -598,11 +617,11 @@ class Remote(LazyMixin, Iterable): progress_handler = progress.new_message_handler() + stderr_text = None + for line in proc.stderr: line = force_text(line) for pline in progress_handler(line): - if line.startswith('fatal:') or line.startswith('error:'): - raise GitCommandError(("Error when fetching: %s" % line,), 2) # END handle special messages for cmd in cmds: if len(line) > 1 and line[0] == ' ' and line[1] == cmd: @@ -612,7 +631,10 @@ class Remote(LazyMixin, Iterable): # end for each comand code we know # end for each line progress didn't handle # end - finalize_process(proc) + if progress.error_lines(): + stderr_text = '\n'.join(progress.error_lines()) + + finalize_process(proc, stderr=stderr_text) # read head information fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb') @@ -626,7 +648,7 @@ class Remote(LazyMixin, Iterable): msg += "length of progress lines %i should be equal to lines in FETCH_HEAD file %i\n" msg += "Will ignore extra progress lines or fetch head lines." msg %= (l_fil, l_fhi) - log.warn(msg) + log.debug(msg) if l_fil < l_fhi: fetch_head_info = fetch_head_info[:l_fil] else: @@ -639,6 +661,8 @@ class Remote(LazyMixin, Iterable): return output def _get_push_info(self, proc, progress): + progress = to_progress_instance(progress) + # read progress information from stderr # we hope stdout can hold all the data, it should ... # read the lines manually as it will use carriage returns between the messages @@ -713,7 +737,7 @@ class Remote(LazyMixin, Iterable): proc = self.repo.git.fetch(self, *args, as_process=True, with_stdout=False, universal_newlines=True, v=True, **kwargs) - res = self._get_fetch_info_from_stderr(proc, progress or RemoteProgress()) + res = self._get_fetch_info_from_stderr(proc, progress) if hasattr(self.repo.odb, 'update_cache'): self.repo.odb.update_cache() return res @@ -732,7 +756,7 @@ class Remote(LazyMixin, Iterable): kwargs = add_progress(kwargs, self.repo.git, progress) proc = self.repo.git.pull(self, refspec, with_stdout=False, as_process=True, universal_newlines=True, v=True, **kwargs) - res = self._get_fetch_info_from_stderr(proc, progress or RemoteProgress()) + res = self._get_fetch_info_from_stderr(proc, progress) if hasattr(self.repo.odb, 'update_cache'): self.repo.odb.update_cache() return res @@ -742,10 +766,19 @@ class Remote(LazyMixin, Iterable): :param refspec: see 'fetch' method :param progress: - Instance of type RemoteProgress allowing the caller to receive - progress information until the method returns. - If None, progress information will be discarded - + Can take one of many value types: + + * None to discard progress information + * A function (callable) that is called with the progress infomation. + + Signature: ``progress(op_code, cur_count, max_count=None, message='')``. + + `Click here <http://goo.gl/NPa7st>`_ for a description of all arguments + given to the function. + * An instance of a class derived from ``git.RemoteProgress`` that + overrides the ``update()`` function. + + :note: No further progress information is returned after push returns. :param kwargs: Additional arguments to be passed to git-push :return: IterableList(PushInfo, ...) iterable list of PushInfo instances, each @@ -758,7 +791,7 @@ class Remote(LazyMixin, Iterable): kwargs = add_progress(kwargs, self.repo.git, progress) proc = self.repo.git.push(self, refspec, porcelain=True, as_process=True, universal_newlines=True, **kwargs) - return self._get_push_info(proc, progress or RemoteProgress()) + return self._get_push_info(proc, progress) @property def config_reader(self): diff --git a/git/repo/base.py b/git/repo/base.py index bc5a7c35..282dfc15 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -32,7 +32,8 @@ from git.index import IndexFile from git.config import GitConfigParser from git.remote import ( Remote, - add_progress + add_progress, + to_progress_instance ) from git.db import GitCmdObjectDB @@ -872,6 +873,9 @@ class Repo(object): @classmethod def _clone(cls, git, url, path, odb_default_type, progress, **kwargs): + if progress is not None: + progress = to_progress_instance(progress) + # special handling for windows for path at which the clone should be # created. # tilde '~' will be expanded to the HOME no matter where the ~ occours. Hence diff --git a/git/test/fixtures/commit_invalid_data b/git/test/fixtures/commit_invalid_data new file mode 100644 index 00000000..d112bf2d --- /dev/null +++ b/git/test/fixtures/commit_invalid_data @@ -0,0 +1,6 @@ +tree 9f1a495d7d9692d24f5caedaa89f5c2c32d59368 +parent 492ace2ffce0e426ebeb55e364e987bcf024dd3b +author E.Azer KoÃoÃoÃoculu <azer@kodfabrik.com> 1306710073 +0300 +committer E.Azer KoÃoÃoÃoculu <azer@kodfabrik.com> 1306710073 +0300 + +add environjs diff --git a/git/test/fixtures/diff_patch_unsafe_paths b/git/test/fixtures/diff_patch_unsafe_paths index 14375f79..1aad6754 100644 --- a/git/test/fixtures/diff_patch_unsafe_paths +++ b/git/test/fixtures/diff_patch_unsafe_paths @@ -61,6 +61,20 @@ index 0000000000000000000000000000000000000000..eaf5f7510320b6a327fb308379de2f94 +++ "b/path/¯\\_(ツ)_|¯" @@ -0,0 +1 @@ +dummy content +diff --git "a/path/\360\237\222\251.txt" "b/path/\360\237\222\251.txt" +new file mode 100644 +index 0000000000000000000000000000000000000000..eaf5f7510320b6a327fb308379de2f94d8859a54 +--- /dev/null ++++ "b/path/\360\237\222\251.txt" +@@ -0,0 +1 @@ ++dummy content +diff --git "a/path/\200-invalid-unicode-path.txt" "b/path/\200-invalid-unicode-path.txt" +new file mode 100644 +index 0000000000000000000000000000000000000000..eaf5f7510320b6a327fb308379de2f94d8859a54 +--- /dev/null ++++ "b/path/\200-invalid-unicode-path.txt" +@@ -0,0 +1 @@ ++dummy content diff --git a/a/with spaces b/b/with some spaces similarity index 100% rename from a/with spaces diff --git a/git/test/test_commit.py b/git/test/test_commit.py index 23b7154a..ea8cd9af 100644 --- a/git/test/test_commit.py +++ b/git/test/test_commit.py @@ -306,6 +306,13 @@ class TestCommit(TestBase): # it appears cmt.author.__repr__() + def test_invalid_commit(self): + cmt = self.rorepo.commit() + cmt._deserialize(open(fixture_path('commit_invalid_data'), 'rb')) + + assert cmt.author.name == u'E.Azer Ko�o�o�oculu', cmt.author.name + assert cmt.author.email == 'azer@kodfabrik.com', cmt.author.email + def test_gpgsig(self): cmt = self.rorepo.commit() cmt._deserialize(open(fixture_path('commit_with_gpgsig'), 'rb')) diff --git a/git/test/test_diff.py b/git/test/test_diff.py index 1d7a4fda..8d189b12 100644 --- a/git/test/test_diff.py +++ b/git/test/test_diff.py @@ -161,16 +161,18 @@ class TestDiff(TestBase): self.assertEqual(res[6].b_path, u'path/with spaces') self.assertEqual(res[7].b_path, u'path/with-question-mark?') self.assertEqual(res[8].b_path, u'path/¯\\_(ツ)_|¯') + self.assertEqual(res[9].b_path, u'path/💩.txt') + self.assertEqual(res[10].b_path, u'path/�-invalid-unicode-path.txt') # The "Moves" # NOTE: The path prefixes a/ and b/ here are legit! We're actually # verifying that it's not "a/a/" that shows up, see the fixture data. - self.assertEqual(res[9].a_path, u'a/with spaces') # NOTE: path a/ here legit! - self.assertEqual(res[9].b_path, u'b/with some spaces') # NOTE: path b/ here legit! - self.assertEqual(res[10].a_path, u'a/ending in a space ') - self.assertEqual(res[10].b_path, u'b/ending with space ') - self.assertEqual(res[11].a_path, u'a/"with-quotes"') - self.assertEqual(res[11].b_path, u'b/"with even more quotes"') + self.assertEqual(res[11].a_path, u'a/with spaces') # NOTE: path a/ here legit! + self.assertEqual(res[11].b_path, u'b/with some spaces') # NOTE: path b/ here legit! + self.assertEqual(res[12].a_path, u'a/ending in a space ') + self.assertEqual(res[12].b_path, u'b/ending with space ') + self.assertEqual(res[13].a_path, u'a/"with-quotes"') + self.assertEqual(res[13].b_path, u'b/"with even more quotes"') def test_diff_patch_format(self): # test all of the 'old' format diffs for completness - it should at least diff --git a/git/test/test_docs.py b/git/test/test_docs.py index 7b3b7474..8dc08559 100644 --- a/git/test/test_docs.py +++ b/git/test/test_docs.py @@ -11,7 +11,6 @@ from gitdb.test.lib import with_rw_directory class Tutorials(TestBase): - @with_rw_directory def test_init_repo_object(self, rw_dir): # [1-test_init_repo_object] @@ -165,7 +164,7 @@ class Tutorials(TestBase): for sm in cloned_repo.submodules: assert not sm.remove().exists() # after removal, the sm doesn't exist anymore sm = cloned_repo.create_submodule('mysubrepo', 'path/to/subrepo', url=bare_repo.git_dir, branch='master') - + # .gitmodules was written and added to the index, which is now being committed cloned_repo.index.commit("Added submodule") assert sm.exists() and sm.module_exists() # this submodule is defintely available @@ -395,7 +394,7 @@ class Tutorials(TestBase): hcommit.diff() # diff tree against index hcommit.diff('HEAD~1') # diff tree against previous tree hcommit.diff(None) # diff tree against working tree - + index = repo.index index.diff() # diff index against itself yielding empty diff index.diff(None) # diff index against working copy @@ -446,7 +445,7 @@ class Tutorials(TestBase): sm = sms[0] assert sm.name == 'gitdb' # git-python has gitdb as single submodule ... assert sm.children()[0].name == 'smmap' # ... which has smmap as single submodule - + # The module is the repository referenced by the submodule assert sm.module_exists() # the module is available, which doesn't have to be the case. assert sm.module().working_tree_dir.endswith('gitdb') @@ -458,7 +457,7 @@ class Tutorials(TestBase): assert sm.config_reader().get_value('path') == sm.path assert len(sm.children()) == 1 # query the submodule hierarchy # ![1-test_submodules] - + @with_rw_directory def test_add_file_and_commit(self, rw_dir): import git diff --git a/git/test/test_git.py b/git/test/test_git.py index 2d6ca8bc..b46ac72d 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -210,7 +210,6 @@ class TestGit(TestBase): assert err.status == 128 else: assert 'FOO' in str(err) - assert err.status == 2 # end # end # end if select.poll exists diff --git a/git/util.py b/git/util.py index a267f183..f5c69231 100644 --- a/git/util.py +++ b/git/util.py @@ -39,7 +39,7 @@ from gitdb.util import ( # NOQA __all__ = ("stream_copy", "join_path", "to_native_path_windows", "to_native_path_linux", "join_path_native", "Stats", "IndexFileSHA1Writer", "Iterable", "IterableList", "BlockingLockFile", "LockFile", 'Actor', 'get_user_id', 'assure_directory_exists', - 'RemoteProgress', 'rmtree', 'WaitGroup', 'unbare_repo') + 'RemoteProgress', 'CallableRemoteProgress', 'rmtree', 'WaitGroup', 'unbare_repo') #{ Utility Methods @@ -160,7 +160,6 @@ def finalize_process(proc, **kwargs): class RemoteProgress(object): - """ Handler providing an interface to parse progress information emitted by git-push and git-fetch and to dispatch callbacks allowing subclasses to react to the progress. @@ -174,23 +173,35 @@ class RemoteProgress(object): DONE_TOKEN = 'done.' TOKEN_SEPARATOR = ', ' - __slots__ = ("_cur_line", "_seen_ops") + __slots__ = ("_cur_line", "_seen_ops", "_error_lines") re_op_absolute = re.compile(r"(remote: )?([\w\s]+):\s+()(\d+)()(.*)") re_op_relative = re.compile(r"(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)") def __init__(self): self._seen_ops = list() self._cur_line = None + self._error_lines = [] + + def error_lines(self): + """Returns all lines that started with error: or fatal:""" + return self._error_lines def _parse_progress_line(self, line): """Parse progress information from the given line as retrieved by git-push - or git-fetch + or git-fetch. + + Lines that seem to contain an error (i.e. start with error: or fatal:) are stored + separately and can be queried using `error_lines()`. :return: list(line, ...) list of lines that could not be processed""" # handle # Counting objects: 4, done. # Compressing objects: 50% (1/2) \rCompressing objects: 100% (2/2) \rCompressing objects: 100% (2/2), done. self._cur_line = line + if len(self._error_lines) > 0 or self._cur_line.startswith(('error:', 'fatal:')): + self._error_lines.append(self._cur_line) + return [] + sub_lines = line.split('\r') failed_lines = list() for sline in sub_lines: @@ -313,10 +324,21 @@ class RemoteProgress(object): You may read the contents of the current line in self._cur_line""" pass + +class CallableRemoteProgress(RemoteProgress): + """An implementation forwarding updates to any callable""" + __slots__ = ('_callable') + + def __init__(self, fn): + self._callable = fn + super(CallableRemoteProgress, self).__init__() + + def update(self, *args, **kwargs): + self._callable(*args, **kwargs) -class Actor(object): +class Actor(object): """Actors hold information about a person acting on the repository. They can be committers and authors or anything with a name and an email as mentioned in the git log entries.""" @@ -754,7 +776,7 @@ class WaitGroup(object): self.cv.notify_all() self.cv.release() - def wait(self): + def wait(self, stderr=b''): self.cv.acquire() while self.count > 0: self.cv.wait() |