#!

/usr/bin/env python3

"""
git-filter-repo filters git repositories, similar to git filter-branch, BFG
repo cleaner, and others. The basic idea is that it works by running
git fast-export <options> | filter | git fast-import <options>
where this program not only launches the whole pipeline but also serves as
the 'filter' in the middle. It does a few additional things on top as well
in order to make it into a well-rounded filtering tool.

git-filter-repo can also be used as a library for more involved filtering

operations; however:
***** API BACKWARD COMPATIBILITY CAVEAT *****
Programs using git-filter-repo as a library can reach pretty far into its
internals, but I am not prepared to guarantee backward compatibility of
all APIs. I suspect changes will be rare, but I reserve the right to
change any API. Since it is assumed that repository filtering is
something one would do very rarely, and in particular that it's a
one-shot operation, this should not be a problem in practice for anyone.
However, if you want to re-use a program you have written that uses
git-filter-repo as a library (or makes use of one of its --*-callback
arguments), you should either make sure you are using the same version of
git and git-filter-repo, or make sure to re-test it.

If there are particular pieces of the API you are concerned about, and
there is not already a testcase for it in t9391-lib-usage.sh or
t9392-python-callback.sh, please contribute a testcase. That will not
prevent me from changing the API, but it will allow you to look at the
history of a testcase to see whether and how the API changed.
***** END API BACKWARD COMPATIBILITY CAVEAT *****
"""

import argparse
import collections
import fnmatch
import gettext
import io
import os
import platform
import re
import shutil
import subprocess
import sys
import time
import textwrap

from datetime import tzinfo, timedelta, datetime

__all__ = ["Blob", "Reset", "FileChange", "Commit", "Tag", "Progress",

"Checkpoint", "FastExportParser", "ProgressWriter",
"string_to_date", "date_to_string",
"record_id_rename", "GitUtils", "FilteringOptions", "RepoFilter"]

# The globals to make visible to callbacks. They will see all our imports for
# free, as well as our public API.
public_globals = ["__builtins__", "argparse", "collections", "fnmatch",
"gettext", "io", "os", "platform", "re", "shutil",
"subprocess", "sys", "time", "textwrap", "tzinfo",
"timedelta", "datetime"] + __all__
deleted_hash = b'0'*40
write_marks = True
date_format_permissive = True

def gettext_poison(msg):
if "GIT_TEST_GETTEXT_POISON" in os.environ: # pragma: no cover
return "# GETTEXT POISON #"
return gettext.gettext(msg)

_ = gettext_poison

def setup_gettext():
TEXTDOMAIN="git-filter-repo"
podir = os.environ.get("GIT_TEXTDOMAINDIR") or "@@LOCALEDIR@@"
if not os.path.isdir(podir): # pragma: no cover
podir = None # Python has its own fallback; use that

## This looks like the most straightforward translation of the relevant

## code in git.git:gettext.c and git.git:perl/Git/I18n.pm:
#import locale
#locale.setlocale(locale.LC_MESSAGES, "");
#locale.setlocale(locale.LC_TIME, "");
#locale.textdomain(TEXTDOMAIN);
#locale.bindtextdomain(TEXTDOMAIN, podir);
## but the python docs suggest using the gettext module (which doesn't
## have setlocale()) instead, so:
gettext.textdomain(TEXTDOMAIN);
gettext.bindtextdomain(TEXTDOMAIN, podir);

def _timedelta_to_seconds(delta):
"""
Converts timedelta to seconds
"""
offset = delta.days*86400 + delta.seconds + (delta.microseconds+0.0)/1000000
return round(offset)

class FixedTimeZone(tzinfo):
"""
Fixed offset in minutes east from UTC.
"""

tz_re = re.compile(br'^([-+]?)(\d\d)(\d\d)$')

def __init__(self, offset_string):

tzinfo.__init__(self)
sign, hh, mm = FixedTimeZone.tz_re.match(offset_string).groups()
factor = -1 if (sign and sign == b'-') else 1
self._offset = timedelta(minutes = factor*(60*int(hh) + int(mm)))
self._offset_string = offset_string

def utcoffset(self, dt):

return self._offset

def tzname(self, dt):

return self._offset_string

def dst(self, dt):

return timedelta(0)
def string_to_date(datestring):
(unix_timestamp, tz_offset) = datestring.split()
return datetime.fromtimestamp(int(unix_timestamp),
FixedTimeZone(tz_offset))

def date_to_string(dateobj):
epoch = datetime.fromtimestamp(0, dateobj.tzinfo)
return(b'%d %s' % (int(_timedelta_to_seconds(dateobj - epoch)),
dateobj.tzinfo.tzname(0)))

def decode(bytestr):
'Try to convert bytestr to utf-8 for outputting as an error message.'
return bytestr.decode('utf-8', 'backslashreplace')

def glob_to_regex(glob_bytestr):
'Translate glob_bytestr into a regex on bytestrings'

# fnmatch.translate is idiotic and won't accept bytestrings

if (decode(glob_bytestr).encode() != glob_bytestr): # pragma: no cover
raise SystemExit(_("Error: Cannot handle glob %s").format(glob_bytestr))

# Create regex operating on string

regex = fnmatch.translate(decode(glob_bytestr))

# FIXME: This is an ugly hack...

# fnmatch.translate tries to do multi-line matching and wants the glob to
# match up to the end of the input, which isn't relevant for us, so we
# have to modify the regex. fnmatch.translate has used different regex
# constructs to achieve this with different python versions, so we have
# to check for each of them and then fix it up. It would be much better
# if fnmatch.translate could just take some flags to allow us to specify
# what we want rather than employing this hackery, but since it
# doesn't...
if regex.endswith(r'\Z(?ms)'): # pragma: no cover
regex = regex[0:-7]
elif regex.startswith(r'(?s:') and regex.endswith(r')\Z'): # pragma: no cover
regex = regex[4:-3]

# Finally, convert back to regex operating on bytestr

return regex.encode()

class PathQuoting:
_unescape = {b'a': b'\a',
b'b': b'\b',
b'f': b'\f',
b'n': b'\n',
b'r': b'\r',
b't': b'\t',
b'v': b'\v',
b'"': b'"',
b'\\':b'\\'}
_unescape_re = re.compile(br'\\([a-z"\\]|[0-9]{3})')
_escape = [bytes([x]) for x in range(127)]+[
b'\\'+bytes(ord(c) for c in oct(x)[2:]) for x in range(127,256)]
_reverse = dict(map(reversed, _unescape.items()))
for x in _reverse:
_escape[ord(x)] = b'\\'+_reverse[x]
_special_chars = [len(x) > 1 for x in _escape]
 @staticmethod
def unescape_sequence(orig):
seq = orig.group(1)
return PathQuoting._unescape[seq] if len(seq) == 1 else bytes([int(seq, 8)])

@staticmethod
def dequote(quoted_string):
if quoted_string.startswith(b'"'):
assert quoted_string.endswith(b'"')
return PathQuoting._unescape_re.sub(PathQuoting.unescape_sequence,
quoted_string[1:-1])
return quoted_string

@staticmethod
def enquote(unquoted_string):
# Option 1: Quoting when fast-export would:
# pqsc = PathQuoting._special_chars
# if any(pqsc[x] for x in set(unquoted_string)):
# Option 2, perf hack: do minimal amount of quoting required by fast-import
if unquoted_string.startswith(b'"') or b'\n' in unquoted_string:
pqe = PathQuoting._escape
return b'"' + b''.join(pqe[x] for x in unquoted_string) + b'"'
return unquoted_string

class AncestryGraph(object):
"""
A class that maintains a direct acycle graph of commits for the purpose of
determining if one commit is the ancestor of another.

A note about identifiers in Commit objects:

* Commit objects have 2 identifiers: commit.old_id and commit.id, because:
* The original fast-export stream identified commits by an identifier.
This is often an integer, but is sometimes a hash (particularly when
--reference-excluded-parents is provided)
* The new fast-import stream we use may not use the same identifiers.
If new blobs or commits are inserted (such as lint-history does), then
the integer (or hash) are no longer valid.

A note about identifiers in AncestryGraph objects, of which there are three:

* A given AncestryGraph is based on either commit.old_id or commit.id, but
not both. These are the keys for self.value.
* Using full hashes (occasionally) for children in self.graph felt
wasteful, so we use our own internal integer within self.graph.
self.value maps from commit {old_}id to our internal integer id.
* When working with commit.old_id, it is also sometimes useful to be able
to map these to the original hash, i.e. commit.original_id. So, we
also have self.git_hash for mapping from commit.old_id to git's commit
hash.
"""

def __init__(self):
# The next internal identifier we will use; increments with every commit
# added to the AncestryGraph
self.cur_value = 0

# A mapping from the external identifers given to us to the simple integers

# we use in self.graph
self.value = {}
 # A tuple of (depth, list-of-ancestors). Values and keys in this graph are
# all integers from the (values of the) self.value dict. The depth of a
# commit is one more than the max depth of any of its ancestors.
self.graph = {}

# A mapping from external identifier (i.e. from the keys of self.value) to

# the hash of the given commit. Only populated for graphs based on
# commit.old_id, since we won't know until later what the git_hash for
# graphs based on commit.id (since we have to wait for fast-import to
# create the commit and notify us of its hash; see _pending_renames).
# elsewhere
self.git_hash = {}

# Reverse maps; only populated if needed. Caller responsible to check

# and ensure they are populated
self._reverse_value = {}
self._hash_to_id = {}

# Cached results from previous calls to is_ancestor().

self._cached_is_ancestor = {}

def record_external_commits(self, external_commits):

"""
Record in graph that each commit in external_commits exists, and is
treated as a root commit with no parents.
"""
for c in external_commits:
if c not in self.value:
self.cur_value += 1
self.value[c] = self.cur_value
self.graph[self.cur_value] = (1, [])
self.git_hash[c] = c

def add_commit_and_parents(self, commit, parents, githash = None):

"""
Record in graph that commit has the given parents (all identified by
fast export stream identifiers, usually integers but sometimes hashes).
parents _MUST_ have been first recorded. commit _MUST_ not have been
recorded yet. Also, record the mapping between commit and githash, if
githash is given.
"""
assert all(p in self.value for p in parents)
assert commit not in self.value

# Get values for commit and parents

self.cur_value += 1
self.value[commit] = self.cur_value
if githash:
self.git_hash[commit] = githash
graph_parents = [self.value[x] for x in parents]

# Determine depth for commit, then insert the info into the graph
depth = 1
if parents:
depth += max(self.graph[p][0] for p in graph_parents)
self.graph[self.cur_value] = (depth, graph_parents)

def record_hash(self, commit_id, githash):

 '''
If a githash was not recorded for commit_id, when add_commit_and_parents
was called, add it now.
'''
assert commit_id in self.value
assert commit_id not in self.git_hash
self.git_hash[commit_id] = githash

def _ensure_reverse_maps_populated(self):
if not self._hash_to_id:
assert not self._reverse_value
self._hash_to_id = {v: k for k, v in self.git_hash.items()}
self._reverse_value = {v: k for k, v in self.value.items()}

def get_parent_hashes(self, commit_hash):

'''
Given a commit_hash, return its parents hashes
'''
#
# We have to map:
# commit hash -> fast export stream id -> graph id
# then lookup
# parent graph ids for given graph id
# then we need to map
# parent graph ids -> parent fast export ids -> parent commit hashes
#
self._ensure_reverse_maps_populated()
commit_fast_export_id = self._hash_to_id[commit_hash]
commit_graph_id = self.value[commit_fast_export_id]
parent_graph_ids = self.graph[commit_graph_id][1]
parent_fast_export_ids = [self._reverse_value[x] for x in parent_graph_ids]
parent_hashes = [self.git_hash[x] for x in parent_fast_export_ids]
return parent_hashes

def map_to_hash(self, commit_id):

'''
Given a commit (by fast export stream id), return its hash
'''
return self.git_hash.get(commit_id, None)

def is_ancestor(self, possible_ancestor, check):

"""
Return whether possible_ancestor is an ancestor of check
"""
a, b = self.value[possible_ancestor], self.value[check]
original_pair = (a,b)
a_depth = self.graph[a][0]
ancestors = [b]
visited = set()
while ancestors:
ancestor = ancestors.pop()
prev_pair = (a, ancestor)
if prev_pair in self._cached_is_ancestor:
if not self._cached_is_ancestor[prev_pair]:
continue
self._cached_is_ancestor[original_pair] = True
return True
if ancestor in visited:
continue
 visited.add(ancestor)
depth, more_ancestors = self.graph[ancestor]
if ancestor == a:
self._cached_is_ancestor[original_pair] = True
return True
elif depth <= a_depth:
continue
ancestors.extend(more_ancestors)
self._cached_is_ancestor[original_pair] = False
return False

class MailmapInfo(object):
def __init__(self, filename):
self.changes = {}
self._parse_file(filename)

def _parse_file(self, filename):

name_and_email_re = re.compile(br'(.*?)\s*<([^>]*)>\s*')
comment_re = re.compile(br'\s*#.*')
if not os.access(filename, os.R_OK):
raise SystemExit(_("Cannot read %s") % decode(filename))
with open(filename, 'br') as f:
count = 0
for line in f:
count += 1
err = "Unparseable mailmap file: line #{} is bad: {}".format(count, line)
# Remove comments
line = comment_re.sub(b'', line)
# Remove leading and trailing whitespace
line = line.strip()
if not line:
continue

m = name_and_email_re.match(line)
if not m:
raise SystemExit(err)
proper_name, proper_email = m.groups()
if len(line) == m.end():
self.changes[(None, proper_email)] = (proper_name, proper_email)
continue
rest = line[m.end():]
m = name_and_email_re.match(rest)
if m:
commit_name, commit_email = m.groups()
if len(rest) != m.end():
raise SystemExit(err)
else:
commit_name, commit_email = rest, None
self.changes[(commit_name, commit_email)] = (proper_name, proper_email)

def translate(self, name, email):

''' Given a name and email, return the expected new name and email from the
mailmap if there is a translation rule for it, otherwise just return
the given name and email.'''
for old, new in self.changes.items():
old_name, old_email = old
new_name, new_email = new
if (old_email is None or email.lower() == old_email.lower()) and (
name == old_name or not old_name):
 return (new_name or name, new_email or email)
return (name, email)

class ProgressWriter(object):
def __init__(self):
self._last_progress_update = time.time()
self._last_message = None

def show(self, msg):

self._last_message = msg
now = time.time()
if now - self._last_progress_update > .1:
self._last_progress_update = now
sys.stdout.write("\r{}".format(msg))
sys.stdout.flush()

def finish(self):
self._last_progress_update = 0
if self._last_message:
self.show(self._last_message)
sys.stdout.write("\n")

class _IDs(object):
"""
A class that maintains the 'name domain' of all the 'marks' (short int
id for a blob/commit git object). There are two reasons this mechanism
is necessary:
(1) the output text of fast-export may refer to an object using a different
mark than the mark that was assigned to that object using IDS.new().
(This class allows you to translate the fast-export marks, "old" to
the marks assigned from IDS.new(), "new").
(2) when we prune a commit, its "old" id becomes invalid. Any commits
which had that commit as a parent needs to use the nearest unpruned
ancestor as its parent instead.

Note that for purpose (1) above, this typically comes about because the user
manually creates Blob or Commit objects (for insertion into the stream).
It could also come about if we attempt to read the data from two different
repositories and trying to combine the data (git fast-export will number ids
from 1...n, and having two 1's, two 2's, two 3's, causes issues; granted, we
this scheme doesn't handle the two streams perfectly either, but if the first
fast export stream is entirely processed and handled before the second stream
is started, this mechanism may be sufficient to handle it).
"""

def __init__(self):
"""
Init
"""
# The id for the next created blob/commit object
self._next_id = 1

# A map of old-ids to new-ids (1:1 map)

self._translation = {}

# A map of new-ids to every old-id that points to the new-id (1:N map)
self._reverse_translation = {}

def has_renames(self):
 """
Return whether there have been ids remapped to new values
"""
return bool(self._translation)

def new(self):
"""
Should be called whenever a new blob or commit object is created. The
returned value should be used as the id/mark for that object.
"""
rv = self._next_id
self._next_id += 1
return rv

def record_rename(self, old_id, new_id, handle_transitivity = False):

"""
Record that old_id is being renamed to new_id.
"""
if old_id != new_id or old_id in self._translation:
# old_id -> new_id
self._translation[old_id] = new_id

# Transitivity will be needed if new commits are being inserted mid-way

# through a branch.
if handle_transitivity:
# Anything that points to old_id should point to new_id
if old_id in self._reverse_translation:
for id_ in self._reverse_translation[old_id]:
self._translation[id_] = new_id

# Record that new_id is pointed to by old_id

if new_id not in self._reverse_translation:
self._reverse_translation[new_id] = []
self._reverse_translation[new_id].append(old_id)

def translate(self, old_id):

"""
If old_id has been mapped to an alternate id, return the alternate id.
"""
if old_id in self._translation:
return self._translation[old_id]
else:
return old_id

def __str__(self):
"""
Convert IDs to string; used for debugging
"""
rv = "Current count: %d\nTranslation:\n" % self._next_id
for k in sorted(self._translation):
rv += " %d -> %s\n" % (k, self._translation[k])

rv += "Reverse translation:\n"
reverse_keys = list(self._reverse_translation.keys())
if None in reverse_keys: # pragma: no cover
reverse_keys.remove(None)
reverse_keys = sorted(reverse_keys)
reverse_keys.append(None)
for k in reverse_keys:
 rv += " " + str(k) + " -> " + str(self._reverse_translation[k]) + "\n"

return rv

class _GitElement(object):
"""
The base class for all git elements that we create.
"""

def __init__(self):
# A string that describes what type of Git element this is
self.type = None

# A flag telling us if this Git element has been dumped

# (i.e. printed) or skipped. Typically elements that have been
# dumped or skipped will not be dumped again.
self.dumped = 0

def dump(self, file_):

"""
This version should never be called. Derived classes need to
override! We should note that subclasses should implement this
method such that the output would match the format produced by
fast-export.
"""
raise SystemExit(_("Unimplemented function: %s") % type(self).__name__
+".dump()") # pragma: no cover

def __bytes__(self):
"""
Convert GitElement to bytestring; used for debugging
"""
old_dumped = self.dumped
writeme = io.BytesIO()
self.dump(writeme)
output_lines = writeme.getvalue().splitlines()
writeme.close()
self.dumped = old_dumped
return b"%s:\n %s" % (type(self).__name__.encode(),
b"\n ".join(output_lines))

def skip(self, new_id=None):

"""
Ensures this element will not be written to output
"""
self.dumped = 2

class _GitElementWithId(_GitElement):
"""
The base class for Git elements that have IDs (commits and blobs)
"""

def __init__(self):
_GitElement.__init__(self)

# The mark (short, portable id) for this element

self.id = _IDS.new()

# The previous mark for this element

 self.old_id = None

def skip(self, new_id=None):

"""
This element will no longer be automatically written to output. When a
commit gets skipped, it's ID will need to be translated to that of its
parent.
"""
self.dumped = 2

_IDS.record_rename(self.old_id or self.id, new_id)

class Blob(_GitElementWithId):
"""
This class defines our representation of git blob elements (i.e. our
way of representing file contents).
"""

def __init__(self, data, original_id = None):

_GitElementWithId.__init__(self)

# Denote that this is a blob

self.type = 'blob'

# Record original id
self.original_id = original_id

# Stores the blob's data

assert(type(data) == bytes)
self.data = data

def dump(self, file_):

"""
Write this blob element to a file.
"""
self.dumped = 1
BLOB_HASH_TO_NEW_ID[self.original_id] = self.id

file_.write(b'blob\n')
file_.write(b'mark :%d\n' % self.id)
file_.write(b'data %d\n%s' % (len(self.data), self.data))
file_.write(b'\n')

class Reset(_GitElement):
"""
This class defines our representation of git reset elements. A reset
event is the creation (or recreation) of a named branch, optionally
starting from a specific revision).
"""

def __init__(self, ref, from_ref = None):

_GitElement.__init__(self)

# Denote that this is a reset

self.type = 'reset'

# The name of the branch being (re)created

self.ref = ref
 # Some reference to the branch/commit we are resetting from
self.from_ref = from_ref

def dump(self, file_):

"""
Write this reset element to a file
"""
self.dumped = 1

file_.write(b'reset %s\n' % self.ref)

if self.from_ref:
if isinstance(self.from_ref, int):
file_.write(b'from :%d\n' % self.from_ref)
else:
file_.write(b'from %s\n' % self.from_ref)
file_.write(b'\n')

class FileChange(_GitElement):
"""
This class defines our representation of file change elements. File change
elements are components within a Commit element.
"""

def __init__(self, type_, filename = None, id_ = None, mode = None):

_GitElement.__init__(self)

# Denote the type of file-change (b'M' for modify, b'D' for delete, etc)
# We could
# assert(type(type_) == bytes)
# here but I don't just due to worries about performance overhead...
self.type = type_

# Record the name of the file being changed

self.filename = filename

# Record the mode (mode describes type of file entry (non-executable,

# executable, or symlink)).
self.mode = mode

# blob_id is the id (mark) of the affected blob

self.blob_id = id_

if type_ == b'DELETEALL':
assert filename is None and id_ is None and mode is None
self.filename = b'' # Just so PathQuoting.enquote doesn't die
else:
assert filename is not None

if type_ == b'M':
assert id_ is not None and mode is not None
elif type_ == b'D':
assert id_ is None and mode is None
elif type_ == b'R': # pragma: no cover (now avoid fast-export renames)
assert mode is None
if id_ is None:
raise SystemExit(_("new name needed for rename of %s") % filename)
self.filename = (self.filename, id_)
self.blob_id = None
 def dump(self, file_):
"""
Write this file-change element to a file
"""
skipped_blob = (self.type == b'M' and self.blob_id is None)
if skipped_blob: return
self.dumped = 1

quoted_filename = PathQuoting.enquote(self.filename)
if self.type == b'M' and isinstance(self.blob_id, int):
file_.write(b'M %s :%d %s\n' % (self.mode, self.blob_id, quoted_filename))
elif self.type == b'M':
file_.write(b'M %s %s %s\n' % (self.mode, self.blob_id, quoted_filename))
elif self.type == b'D':
file_.write(b'D %s\n' % quoted_filename)
elif self.type == b'DELETEALL':
file_.write(b'deleteall\n')
else:
raise SystemExit(_("Unhandled filechange type: %s") % self.type) # pragma: no
cover

class Commit(_GitElementWithId):
"""
This class defines our representation of commit elements. Commit elements
contain all the information associated with a commit.
"""

def __init__(self, branch,

author_name, author_email, author_date,
committer_name, committer_email, committer_date,
message,
file_changes,
parents,
original_id = None,
encoding = None, # encoding for message; None implies UTF-8
**kwargs):
_GitElementWithId.__init__(self)
self.old_id = self.id

# Denote that this is a commit element

self.type = 'commit'

# Record the affected branch

self.branch = branch

# Record original id
self.original_id = original_id

# Record author's name

self.author_name = author_name

# Record author's email

self.author_email = author_email

# Record date of authoring

self.author_date = author_date

# Record committer's name

 self.committer_name = committer_name

# Record committer's email

self.committer_email = committer_email

# Record date the commit was made

self.committer_date = committer_date

# Record commit message and its encoding

self.encoding = encoding
self.message = message

# List of file-changes associated with this commit. Note that file-changes

# are also represented as git elements
self.file_changes = file_changes

self.parents = parents

def dump(self, file_):

"""
Write this commit element to a file.
"""
self.dumped = 1

# Make output to fast-import slightly easier for humans to read if the

# message has no trailing newline of its own; cosmetic, but a nice touch...
extra_newline = b'\n'
if self.message.endswith(b'\n') or not (self.parents or self.file_changes):
extra_newline = b''

if not self.parents:
file_.write(b'reset %s\n' % self.branch)
file_.write((b'commit %s\n'
b'mark :%d\n'
b'author %s <%s> %s\n'
b'committer %s <%s> %s\n'
) % (
self.branch, self.id,
self.author_name, self.author_email, self.author_date,
self.committer_name, self.committer_email, self.committer_date
))
if self.encoding:
file_.write(b'encoding %s\n' % self.encoding)
file_.write(b'data %d\n%s%s' %
(len(self.message), self.message, extra_newline))
for i, parent in enumerate(self.parents):
file_.write(b'from ' if i==0 else b'merge ')
if isinstance(parent, int):
file_.write(b':%d\n' % parent)
else:
file_.write(b'%s\n' % parent)
for change in self.file_changes:
change.dump(file_)
if not self.parents and not self.file_changes:
# Workaround a bug in pre-git-2.22 versions of fast-import with
# the get-mark directive.
file_.write(b'\n')
file_.write(b'\n')
 def first_parent(self):
"""
Return first parent commit
"""
if self.parents:
return self.parents[0]
return None

def skip(self, new_id=None):

_SKIPPED_COMMITS.add(self.old_id or self.id)
_GitElementWithId.skip(self, new_id)

class Tag(_GitElementWithId):
"""
This class defines our representation of annotated tag elements.
"""

def __init__(self, ref, from_ref,

tagger_name, tagger_email, tagger_date, tag_msg,
original_id = None):
_GitElementWithId.__init__(self)
self.old_id = self.id

# Denote that this is a tag element

self.type = 'tag'

# Store the name of the tag

self.ref = ref

# Store the entity being tagged (this should be a commit)

self.from_ref = from_ref

# Record original id
self.original_id = original_id

# Store the name of the tagger

self.tagger_name = tagger_name

# Store the email of the tagger

self.tagger_email = tagger_email

# Store the date

self.tagger_date = tagger_date

# Store the tag message

self.message = tag_msg

def dump(self, file_):

"""
Write this tag element to a file
"""

self.dumped = 1

file_.write(b'tag %s\n' % self.ref)

if (write_marks and self.id):
file_.write(b'mark :%d\n' % self.id)
markfmt = b'from :%d\n' if isinstance(self.from_ref, int) else b'from %s\n'
file_.write(markfmt % self.from_ref)
 if self.tagger_name:
file_.write(b'tagger %s <%s> ' % (self.tagger_name, self.tagger_email))
file_.write(self.tagger_date)
file_.write(b'\n')
file_.write(b'data %d\n%s' % (len(self.message), self.message))
file_.write(b'\n')

class Progress(_GitElement):
"""
This class defines our representation of progress elements. The progress
element only contains a progress message, which is printed by fast-import
when it processes the progress output.
"""

def __init__(self, message):

_GitElement.__init__(self)

# Denote that this is a progress element

self.type = 'progress'

# Store the progress message

self.message = message

def dump(self, file_):

"""
Write this progress element to a file
"""
self.dumped = 1

file_.write(b'progress %s\n' % self.message)

file_.write(b'\n')

class Checkpoint(_GitElement):
"""
This class defines our representation of checkpoint elements. These
elements represent events which force fast-import to close the current
packfile, start a new one, and to save out all current branch refs, tags
and marks.
"""

def __init__(self):
_GitElement.__init__(self)

# Denote that this is a checkpoint element

self.type = 'checkpoint'

def dump(self, file_):

"""
Write this checkpoint element to a file
"""
self.dumped = 1

file_.write(b'checkpoint\n')
file_.write(b'\n')

class LiteralCommand(_GitElement):
"""
This class defines our representation of commands. The literal command
includes only a single line, and is not processed in any special way.
 """

def __init__(self, line):

_GitElement.__init__(self)

# Denote that this is a literal element

self.type = 'literal'

# Store the command

self.line = line

def dump(self, file_):

"""
Write this progress element to a file
"""
self.dumped = 1

file_.write(self.line)

class Alias(_GitElement):
"""
This class defines our representation of fast-import alias elements. An
alias element is the setting of one mark to the same sha1sum as another,
usually because the newer mark corresponded to a pruned commit.
"""

def __init__(self, ref, to_ref):

_GitElement.__init__(self)
# Denote that this is a reset
self.type = 'alias'

self.ref = ref
self.to_ref = to_ref

def dump(self, file_):

"""
Write this reset element to a file
"""
self.dumped = 1

file_.write(b'alias\nmark :%d\nto :%d\n\n' % (self.ref, self.to_ref))

class FastExportParser(object):
"""
A class for parsing and handling the output from fast-export. This
class allows the user to register callbacks when various types of
data are encountered in the fast-export output. The basic idea is that,
FastExportParser takes fast-export output, creates the various objects
as it encounters them, the user gets to use/modify these objects via
callbacks, and finally FastExportParser outputs the modified objects
in fast-import format (presumably so they can be used to create a new
repo).
"""

def __init__(self,
tag_callback = None, commit_callback = None,
blob_callback = None, progress_callback = None,
reset_callback = None, checkpoint_callback = None,
done_callback = None):
 # Members below simply store callback functions for the various git
# elements
self._tag_callback = tag_callback
self._blob_callback = blob_callback
self._reset_callback = reset_callback
self._commit_callback = commit_callback
self._progress_callback = progress_callback
self._checkpoint_callback = checkpoint_callback
self._done_callback = done_callback

# Keep track of which refs appear from the export, and which make it to
# the import (pruning of empty commits, renaming of refs, and creating
# new manual objects and inserting them can cause these to differ).
self._exported_refs = set()
self._imported_refs = set()

# A list of the branches we've seen, plus the last known commit they
# pointed to. An entry in latest_*commit will be deleted if we get a
# reset for that branch. These are used because of fast-import's weird
# decision to allow having an implicit parent via naming the branch
# instead of requiring branches to be specified via 'from' directives.
self._latest_commit = {}
self._latest_orig_commit = {}

# A handle to the input source for the fast-export data

self._input = None

# A handle to the output file for the output we generate (we call dump
# on many of the git elements we create).
self._output = None

# Stores the contents of the current line of input being parsed

self._currentline = ''

# Compile some regexes and cache those

self._mark_re = re.compile(br'mark :(\d+)\n$')
self._parent_regexes = {}
parent_regex_rules = (br' :(\d+)\n$', br' ([0-9a-f]{40})\n')
for parent_refname in (b'from', b'merge'):
ans = [re.compile(parent_refname+x) for x in parent_regex_rules]
self._parent_regexes[parent_refname] = ans
self._quoted_string_re = re.compile(br'"(?:[^"\\]|\\.)*"')
self._refline_regexes = {}
for refline_name in (b'reset', b'commit', b'tag', b'progress'):
self._refline_regexes[refline_name] = re.compile(refline_name+b' (.*)\n$')
self._user_regexes = {}
for user in (b'author', b'committer', b'tagger'):
self._user_regexes[user] = re.compile(user + b' (.*?) <(.*?)> (.*)\n$')

def _advance_currentline(self):
"""
Grab the next line of input
"""
self._currentline = self._input.readline()

def _parse_optional_mark(self):
"""
If the current line contains a mark, parse it and advance to the
next line; return None otherwise
 """
mark = None
matches = self._mark_re.match(self._currentline)
if matches:
mark = int(matches.group(1))
self._advance_currentline()
return mark

def _parse_optional_parent_ref(self, refname):

"""
If the current line contains a reference to a parent commit, then
parse it and advance the current line; otherwise return None. Note
that the name of the reference ('from', 'merge') must match the
refname arg.
"""
orig_baseref, baseref = None, None
rule, altrule = self._parent_regexes[refname]
matches = rule.match(self._currentline)
if matches:
orig_baseref = int(matches.group(1))
# We translate the parent commit mark to what it needs to be in
# our mark namespace
baseref = _IDS.translate(orig_baseref)
self._advance_currentline()
else:
matches = altrule.match(self._currentline)
if matches:
orig_baseref = matches.group(1)
baseref = orig_baseref
self._advance_currentline()
return orig_baseref, baseref

def _parse_optional_filechange(self):
"""
If the current line contains a file-change object, then parse it
and advance the current line; otherwise return None. We only care
about file changes of type b'M' and b'D' (these are the only types
of file-changes that fast-export will provide).
"""
filechange = None
changetype = self._currentline[0:1]
if changetype == b'M':
(changetype, mode, idnum, path) = self._currentline.split(None, 3)
if idnum[0:1] == b':':
idnum = idnum[1:]
path = path.rstrip(b'\n')
# We translate the idnum to our id system
if len(idnum) != 40:
idnum = _IDS.translate( int(idnum) )
if idnum is not None:
if path.startswith(b'"'):
path = PathQuoting.dequote(path)
filechange = FileChange(b'M', path, idnum, mode)
else:
filechange = b'skipped'
self._advance_currentline()
elif changetype == b'D':
(changetype, path) = self._currentline.split(None, 1)
path = path.rstrip(b'\n')
 if path.startswith(b'"'):
path = PathQuoting.dequote(path)
filechange = FileChange(b'D', path)
self._advance_currentline()
elif changetype == b'R': # pragma: no cover (now avoid fast-export renames)
rest = self._currentline[2:-1]
if rest.startswith(b'"'):
m = self._quoted_string_re.match(rest)
if not m:
raise SystemExit(_("Couldn't parse rename source"))
orig = PathQuoting.dequote(m.group(0))
new = rest[m.end()+1:]
else:
orig, new = rest.split(b' ', 1)
if new.startswith(b'"'):
new = PathQuoting.dequote(new)
filechange = FileChange(b'R', orig, new)
self._advance_currentline()
return filechange

def _parse_original_id(self):
original_id = self._currentline[len(b'original-oid '):].rstrip()
self._advance_currentline()
return original_id

def _parse_encoding(self):
encoding = self._currentline[len(b'encoding '):].rstrip()
self._advance_currentline()
return encoding

def _parse_ref_line(self, refname):

"""
Parses string data (often a branch name) from current-line. The name of
the string data must match the refname arg. The program will crash if
current-line does not match, so current-line will always be advanced if
this method returns.
"""
matches = self._refline_regexes[refname].match(self._currentline)
if not matches:
raise SystemExit(_("Malformed %(refname)s line: '%(line)s'") %
({'refname': refname, 'line':self._currentline})
) # pragma: no cover
ref = matches.group(1)
self._advance_currentline()
return ref

def _parse_user(self, usertype):

"""
Get user name, email, datestamp from current-line. Current-line will
be advanced.
"""
user_regex = self._user_regexes[usertype]
(name, email, when) = user_regex.match(self._currentline).groups()

self._advance_currentline()
return (name, email, when)

def _parse_data(self):
"""
 Reads data from _input. Current-line will be advanced until it is beyond
the data.
"""
fields = self._currentline.split()
assert fields[0] == b'data'
size = int(fields[1])
data = self._input.read(size)
self._advance_currentline()
if self._currentline == b'\n':
self._advance_currentline()
return data

def _parse_blob(self):
"""
Parse input data into a Blob object. Once the Blob has been created, it
will be handed off to the appropriate callbacks. Current-line will be
advanced until it is beyond this blob's data. The Blob will be dumped
to _output once everything else is done (unless it has been skipped by
the callback).
"""
# Parse the Blob
self._advance_currentline()
id_ = self._parse_optional_mark()

original_id = None
if self._currentline.startswith(b'original-oid'):
original_id = self._parse_original_id();

data = self._parse_data()
if self._currentline == b'\n':
self._advance_currentline()

# Create the blob

blob = Blob(data, original_id)

# If fast-export text had a mark for this blob, need to make sure this
# mark translates to the blob's true id.
if id_:
blob.old_id = id_
_IDS.record_rename(id_, blob.id)

# Call any user callback to allow them to use/modify the blob

if self._blob_callback:
self._blob_callback(blob)

# Now print the resulting blob

if not blob.dumped:
blob.dump(self._output)

def _parse_reset(self):
"""
Parse input data into a Reset object. Once the Reset has been created,
it will be handed off to the appropriate callbacks. Current-line will
be advanced until it is beyond the reset data. The Reset will be dumped
to _output once everything else is done (unless it has been skipped by
the callback).
"""
# Parse the Reset
ref = self._parse_ref_line(b'reset')
 self._exported_refs.add(ref)
ignoreme, from_ref = self._parse_optional_parent_ref(b'from')
if self._currentline == b'\n':
self._advance_currentline()

# fast-export likes to print extraneous resets that serve no purpose.

# While we could continue processing such resets, that is a waste of
# resources. Also, we want to avoid recording that this ref was
# seen in such cases, since this ref could be rewritten to nothing.
if not from_ref:
self._latest_commit.pop(ref, None)
self._latest_orig_commit.pop(ref, None)
return

# Create the reset

reset = Reset(ref, from_ref)

# Call any user callback to allow them to modify the reset

if self._reset_callback:
self._reset_callback(reset)

# Update metadata
self._latest_commit[reset.ref] = reset.from_ref
self._latest_orig_commit[reset.ref] = reset.from_ref

# Now print the resulting reset

if not reset.dumped:
self._imported_refs.add(reset.ref)
reset.dump(self._output)

def _parse_commit(self):
"""
Parse input data into a Commit object. Once the Commit has been created,
it will be handed off to the appropriate callbacks. Current-line will
be advanced until it is beyond the commit data. The Commit will be dumped
to _output once everything else is done (unless it has been skipped by
the callback OR the callback has removed all file-changes from the commit).
"""
# Parse the Commit. This may look involved, but it's pretty simple; it only
# looks bad because a commit object contains many pieces of data.
branch = self._parse_ref_line(b'commit')
self._exported_refs.add(branch)
id_ = self._parse_optional_mark()

original_id = None
if self._currentline.startswith(b'original-oid'):
original_id = self._parse_original_id();

author_name = None
author_email = None
if self._currentline.startswith(b'author'):
(author_name, author_email, author_date) = self._parse_user(b'author')

(committer_name, committer_email, committer_date) = \

self._parse_user(b'committer')

if not author_name and not author_email:

(author_name, author_email, author_date) = \
(committer_name, committer_email, committer_date)
encoding = None
if self._currentline.startswith(b'encoding '):
encoding = self._parse_encoding()

commit_msg = self._parse_data()

pinfo = [self._parse_optional_parent_ref(b'from')]
# Due to empty pruning, we can have real 'from' and 'merge' lines that
# due to commit rewriting map to a parent of None. We need to record
# 'from' if its non-None, and we need to parse all 'merge' lines.
while self._currentline.startswith(b'merge '):
pinfo.append(self._parse_optional_parent_ref(b'merge'))
orig_parents, parents = [list(tmp) for tmp in zip(*pinfo)]

# No parents is oddly represented as [None] instead of [], due to the

# special 'from' handling. Convert it here to a more canonical form.
if parents == [None]:
parents = []
if orig_parents == [None]:
orig_parents = []

# fast-import format is kinda stupid in that it allows implicit parents

# based on the branch name instead of requiring them to be specified by
# 'from' directives. The only way to get no parent is by using a reset
# directive first, which clears the latest_commit_for_this_branch tracking.
if not orig_parents and self._latest_commit.get(branch):
parents = [self._latest_commit[branch]]
if not orig_parents and self._latest_orig_commit.get(branch):
orig_parents = [self._latest_orig_commit[branch]]

# Get the list of file changes

file_changes = []
file_change = self._parse_optional_filechange()
had_file_changes = file_change is not None
while file_change:
if not (type(file_change) == bytes and file_change == b'skipped'):
file_changes.append(file_change)
file_change = self._parse_optional_filechange()
if self._currentline == b'\n':
self._advance_currentline()

# Okay, now we can finally create the Commit object

commit = Commit(branch,
author_name, author_email, author_date,
committer_name, committer_email, committer_date,
commit_msg, file_changes, parents, original_id, encoding)

# If fast-export text had a mark for this commit, need to make sure this
# mark translates to the commit's true id.
if id_:
commit.old_id = id_
_IDS.record_rename(id_, commit.id)

# Call any user callback to allow them to modify the commit

aux_info = {'orig_parents': orig_parents,
'had_file_changes': had_file_changes}
if self._commit_callback:
self._commit_callback(commit, aux_info)
 # Now print the resulting commit, or if prunable skip it
self._latest_orig_commit[branch] = commit.id
if not (commit.old_id or commit.id) in _SKIPPED_COMMITS:
self._latest_commit[branch] = commit.id
if not commit.dumped:
self._imported_refs.add(commit.branch)
commit.dump(self._output)

def _parse_tag(self):
"""
Parse input data into a Tag object. Once the Tag has been created,
it will be handed off to the appropriate callbacks. Current-line will
be advanced until it is beyond the tag data. The Tag will be dumped
to _output once everything else is done (unless it has been skipped by
the callback).
"""
# Parse the Tag
tag = self._parse_ref_line(b'tag')
self._exported_refs.add(b'refs/tags/'+tag)
id_ = self._parse_optional_mark()
ignoreme, from_ref = self._parse_optional_parent_ref(b'from')

original_id = None
if self._currentline.startswith(b'original-oid'):
original_id = self._parse_original_id();

tagger_name, tagger_email, tagger_date = None, None, None

if self._currentline.startswith(b'tagger'):
(tagger_name, tagger_email, tagger_date) = self._parse_user(b'tagger')
tag_msg = self._parse_data()
if self._currentline == b'\n':
self._advance_currentline()

# Create the tag

tag = Tag(tag, from_ref,
tagger_name, tagger_email, tagger_date, tag_msg,
original_id)

# If fast-export text had a mark for this tag, need to make sure this
# mark translates to the tag's true id.
if id_:
tag.old_id = id_
_IDS.record_rename(id_, tag.id)

# Call any user callback to allow them to modify the tag

if self._tag_callback:
self._tag_callback(tag)

# The tag might not point at anything that still exists (self.from_ref
# will be None if the commit it pointed to and all its ancestors were
# pruned due to being empty)
if tag.from_ref:
# Print out this tag's information
if not tag.dumped:
self._imported_refs.add(b'refs/tags/'+tag.ref)
tag.dump(self._output)
else:
tag.skip()
def _parse_progress(self):
"""
Parse input data into a Progress object. Once the Progress has
been created, it will be handed off to the appropriate
callbacks. Current-line will be advanced until it is beyond the
progress data. The Progress will be dumped to _output once
everything else is done (unless it has been skipped by the callback).
"""
# Parse the Progress
message = self._parse_ref_line(b'progress')
if self._currentline == b'\n':
self._advance_currentline()

# Create the progress message

progress = Progress(message)

# Call any user callback to allow them to modify the progress messsage
if self._progress_callback:
self._progress_callback(progress)

# NOTE: By default, we do NOT print the progress message; git

# fast-import would write it to fast_import_pipes which could mess with
# our parsing of output from the 'ls' and 'get-mark' directives we send
# to fast-import. If users want these messages, they need to process
# and handle them in the appropriate callback above.

def _parse_checkpoint(self):
"""
Parse input data into a Checkpoint object. Once the Checkpoint has
been created, it will be handed off to the appropriate
callbacks. Current-line will be advanced until it is beyond the
checkpoint data. The Checkpoint will be dumped to _output once
everything else is done (unless it has been skipped by the callback).
"""
# Parse the Checkpoint
self._advance_currentline()
if self._currentline == b'\n':
self._advance_currentline()

# Create the checkpoint

checkpoint = Checkpoint()

# Call any user callback to allow them to drop the checkpoint

if self._checkpoint_callback:
self._checkpoint_callback(checkpoint)

# NOTE: By default, we do NOT print the checkpoint message; although it

# we would only realistically get them with --stdin, the fact that we
# are filtering makes me think the checkpointing is less likely to be
# reasonable. In fact, I don't think it's necessary in general. If
# users do want it, they should process it in the checkpoint_callback.

def _parse_literal_command(self):
"""
Parse literal command. Then just dump the line as is.
"""
# Create the literal command object
command = LiteralCommand(self._currentline)
 self._advance_currentline()

# Now print the resulting literal command

if not command.dumped:
command.dump(self._output)

def insert(self, obj):

assert not obj.dumped
obj.dump(self._output)
if type(obj) == Commit:
self._imported_refs.add(obj.branch)
elif type(obj) in (Reset, Tag):
self._imported_refs.add(obj.ref)

def run(self, input, output):

"""
This method filters fast export output.
"""
# Set input. If no args provided, use stdin.
self._input = input
self._output = output

# Run over the input and do the filtering

self._advance_currentline()
while self._currentline:
if self._currentline.startswith(b'blob'):
self._parse_blob()
elif self._currentline.startswith(b'reset'):
self._parse_reset()
elif self._currentline.startswith(b'commit'):
self._parse_commit()
elif self._currentline.startswith(b'tag'):
self._parse_tag()
elif self._currentline.startswith(b'progress'):
self._parse_progress()
elif self._currentline.startswith(b'checkpoint'):
self._parse_checkpoint()
elif self._currentline.startswith(b'feature'):
self._parse_literal_command()
elif self._currentline.startswith(b'option'):
self._parse_literal_command()
elif self._currentline.startswith(b'done'):
if self._done_callback:
self._done_callback()
self._parse_literal_command()
# Prevent confusion from others writing additional stuff that'll just
# be ignored
self._output.close()
elif self._currentline.startswith(b'#'):
self._parse_literal_command()
elif self._currentline.startswith(b'get-mark') or \
self._currentline.startswith(b'cat-blob') or \
self._currentline.startswith(b'ls'):
raise SystemExit(_("Unsupported command: '%s'") % self._currentline)
else:
raise SystemExit(_("Could not parse line: '%s'") % self._currentline)

def get_exported_and_imported_refs(self):
return self._exported_refs, self._imported_refs
def record_id_rename(old_id, new_id):
"""
Register a new translation
"""
handle_transitivity = True
_IDS.record_rename(old_id, new_id, handle_transitivity)

# Internal globals
_IDS = _IDs()
_SKIPPED_COMMITS = set()
BLOB_HASH_TO_NEW_ID = {}

class SubprocessWrapper(object):
@staticmethod
def decodify(args):
if type(args) == str:
return args
else:
assert type(args) == list
return [decode(x) if type(x)==bytes else x for x in args]

@staticmethod
def call(*args, **kwargs):
if 'cwd' in kwargs:
kwargs['cwd'] = decode(kwargs['cwd'])
return subprocess.call(SubprocessWrapper.decodify(*args), **kwargs)

@staticmethod
def check_output(*args, **kwargs):
if 'cwd' in kwargs:
kwargs['cwd'] = decode(kwargs['cwd'])
return subprocess.check_output(SubprocessWrapper.decodify(*args), **kwargs)

@staticmethod
def check_call(*args, **kwargs): # pragma: no cover # used by filter-lamely
if 'cwd' in kwargs:
kwargs['cwd'] = decode(kwargs['cwd'])
return subprocess.check_call(SubprocessWrapper.decodify(*args), **kwargs)

@staticmethod
def Popen(*args, **kwargs):
if 'cwd' in kwargs:
kwargs['cwd'] = decode(kwargs['cwd'])
return subprocess.Popen(SubprocessWrapper.decodify(*args), **kwargs)

subproc = subprocess
if platform.system() == 'Windows' or 'PRETEND_UNICODE_ARGS' in os.environ:
subproc = SubprocessWrapper

class GitUtils(object):
@staticmethod
def get_commit_count(repo, *args):
"""
Return the number of commits that have been made on repo.
"""
if not args:
args = ['--all']
if len(args) == 1 and isinstance(args[0], list):
 args = args[0]
p = subproc.Popen(["git", "rev-list", "--count"] + args,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
cwd=repo)
if p.wait() != 0:
raise SystemExit(_("%s does not appear to be a valid git repository")
% decode(repo))
return int(p.stdout.read())

@staticmethod
def get_total_objects(repo):
"""
Return the number of objects (both packed and unpacked)
"""
p1 = subproc.Popen(["git", "count-objects", "-v"],
stdout=subprocess.PIPE, cwd=repo)
lines = p1.stdout.read().splitlines()
# Return unpacked objects + packed-objects
return int(lines[0].split()[1]) + int(lines[2].split()[1])

@staticmethod
def is_repository_bare(repo_working_dir):
out = subproc.check_output('git rev-parse --is-bare-repository'.split(),
cwd=repo_working_dir)
return (out.strip() == b'true')

@staticmethod
def determine_git_dir(repo_working_dir):
d = subproc.check_output('git rev-parse --git-dir'.split(),
cwd=repo_working_dir).strip()
if repo_working_dir==b'.' or d.startswith(b'/'):
return d
return os.path.join(repo_working_dir, d)

@staticmethod
def get_refs(repo_working_dir):
try:
output = subproc.check_output('git show-ref'.split(),
cwd=repo_working_dir)
except subprocess.CalledProcessError as e:
# If error code is 1, there just aren't any refs; i.e. new repo.
# If error code is other than 1, some other error (e.g. not a git repo)
if e.returncode != 1:
raise SystemExit('fatal: {}'.format(e))
output = ''
return dict(reversed(x.split()) for x in output.splitlines())

@staticmethod
def get_config_settings(repo_working_dir):
output = ''
try:
output = subproc.check_output('git config --list'.split(),
cwd=repo_working_dir)
except subprocess.CalledProcessError as e: # pragma: no cover
raise SystemExit('fatal: {}'.format(e))

# FIXME: Ignores multi-valued keys, just let them overwrite for now
return dict(line.split(b'=', maxsplit=1)
for line in output.strip().split(b"\n"))
 @staticmethod
def get_blob_sizes(quiet = False):
blob_size_progress = ProgressWriter()
num_blobs = 0
processed_blobs_msg = _("Processed %d blob sizes")

# Get sizes of blobs by sha1

cmd = '--batch-check=%(objectname) %(objecttype) ' + \
'%(objectsize) %(objectsize:disk)'
cf = subproc.Popen(['git', 'cat-file', '--batch-all-objects', cmd],
bufsize = -1,
stdout = subprocess.PIPE)
unpacked_size = {}
packed_size = {}
for line in cf.stdout:
try:
sha, objtype, objsize, objdisksize = line.split()
objsize, objdisksize = int(objsize), int(objdisksize)
if objtype == b'blob':
unpacked_size[sha] = objsize
packed_size[sha] = objdisksize
num_blobs += 1
except ValueError: # pragma: no cover
sys.stderr.write(_("Error: unexpected `git cat-file` output: \"%s\"\n") %
line)
if not quiet:
blob_size_progress.show(processed_blobs_msg % num_blobs)
cf.wait()
if not quiet:
blob_size_progress.finish()
return unpacked_size, packed_size

@staticmethod
def get_file_changes(repo, parent_hash, commit_hash):
"""
Return a FileChanges list with the differences between parent_hash
and commit_hash
"""
file_changes = []

cmd = ["git", "diff-tree", "-r", parent_hash, commit_hash]

output = subproc.check_output(cmd, cwd=repo)
for line in output.splitlines():
fileinfo, path = line.split(b'\t', 1)
if path.startswith(b'"'):
path = PathQuoting.dequote(path)
oldmode, mode, oldhash, newhash, changetype = fileinfo.split()
if changetype == b'D':
file_changes.append(FileChange(b'D', path))
elif changetype in (b'A', b'M', b'T'):
identifier = BLOB_HASH_TO_NEW_ID.get(newhash, newhash)
file_changes.append(FileChange(b'M', path, identifier, mode))
else: # pragma: no cover
raise SystemExit("Unknown change type for line {}".format(line))

return file_changes

@staticmethod
 def print_my_version():
with open(__file__, 'br') as f:
contents = f.read()
# If people replaced @@LOCALEDIR@@ string to point at their local
# directory, undo it so we can get original source version.
contents = re.sub(br'\A#\!.*',
br'#!/usr/bin/env python3', contents)
contents = re.sub(br'(\("GIT_TEXTDOMAINDIR"\) or ").*"',
br'\1@@LOCALEDIR@@"', contents)

cmd = 'git hash-object --stdin'.split()

version = subproc.check_output(cmd, input=contents).strip()
print(decode(version[0:12]))

class FilteringOptions(object):
default_replace_text = b'***REMOVED***'
class AppendFilter(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
user_path = values
suffix = option_string[len('--path-'):] or 'match'
if suffix.startswith('rename'):
mod_type = 'rename'
match_type = option_string[len('--path-rename-'):] or 'match'
values = values.split(b':')
if len(values) != 2:
raise SystemExit(_("Error: --path-rename expects one colon in its"
" argument: <old_name:new_name>."))
if values[0] and values[1] and not (
values[0].endswith(b'/') == values[1].endswith(b'/')):
raise SystemExit(_("Error: With --path-rename, if OLD_NAME and "
"NEW_NAME are both non-empty and either ends "
"with a slash then both must."))
if any(v.startswith(b'/') for v in values):
raise SystemExit(_("Error: Pathnames cannot begin with a '/'"))
components = values[0].split(b'/') + values[1].split(b'/')
else:
mod_type = 'filter'
match_type = suffix
components = values.split(b'/')
if values.startswith(b'/'):
raise SystemExit(_("Error: Pathnames cannot begin with a '/'"))
for illegal_path in [b'.', b'..']:
if illegal_path in components:
raise SystemExit(_("Error: Invalid path component '%s' found in '%s'")
% (decode(illegal_path), decode(user_path)))
if match_type == 'regex':
values = re.compile(values)
items = getattr(namespace, self.dest, []) or []
items.append((mod_type, match_type, values))
if (match_type, mod_type) == ('glob', 'filter'):
if not values.endswith(b'*'):
extension = b'*' if values.endswith(b'/') else b'/*'
items.append((mod_type, match_type, values+extension))
setattr(namespace, self.dest, items)

class HelperFilter(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
af = FilteringOptions.AppendFilter(dest='path_changes',
option_strings=None)
 dirname = values if values[-1:] == b'/' else values+b'/'
if option_string == '--subdirectory-filter':
af(parser, namespace, dirname, '--path-match')
af(parser, namespace, dirname+b':', '--path-rename')
elif option_string == '--to-subdirectory-filter':
af(parser, namespace, b':'+dirname, '--path-rename')
else:
raise SystemExit(_("Error: HelperFilter given invalid option_string: %s")
% option_string) # pragma: no cover

class FileWithPathsFilter(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
if not namespace.path_changes:
namespace.path_changes = []
namespace.path_changes += FilteringOptions.get_paths_from_file(values)

@staticmethod
def create_arg_parser():
# Include usage in the summary, so we can put the description first
summary = _('''Rewrite (or analyze) repository history

git-filter-repo destructively rewrites history (unless --analyze or

--dry-run are given) according to specified rules. It refuses to do any
rewriting unless either run from a clean fresh clone, or --force was
given.

Basic Usage:
git-filter-repo --analyze
git-filter-repo [FILTER/RENAME/CONTROL OPTIONS]

See EXAMPLES section for details.

''').rstrip()

# Provide a long helpful examples section

example_text = _('''CALLBACKS

All callback functions are of the same general format. For a command line
argument like
--foo-callback 'BODY'

the following code will be compiled and called:

def foo_callback(foo):
BODY

Thus, to replace 'Jon' with 'John' in author/committer/tagger names:

git filter-repo --name-callback 'return name.replace(b"Jon", b"John")'

To remove all 'Tested-by' tags in commit (or tag) messages:

git filter-repo --message-callback 'return re.sub(br"\\nTested-by:.*", "",
message)'

To remove all .DS_Store files:

git filter-repo --filename-callback 'return None if
os.path.basename(filename) == b".DS_Store" else filename'

Note that if BODY resolves to a filename, then the contents of that file
will be used as the BODY in the callback function.

For more detailed examples and explanations AND caveats, see

 https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/
blob/docs/html/git-filter-repo.html#CALLBACKS

EXAMPLES

To get a bunch of reports mentioning renames that have occurred in

your repo and listing sizes of objects aggregated by any of path,
directory, extension, or blob-id:
git filter-repo --analyze

(These reports can help you choose how to filter your repo; it can
be useful to re-run this command after filtering to regenerate the
report and verify the changes look correct.)

To extract the history that touched just 'guides' and 'tools/releases':

git filter-repo --path guides/ --path tools/releases

To remove foo.zip and bar/baz/zips from every revision in history:

git filter-repo --path foo.zip --path bar/baz/zips/ --invert-paths

To replace the text 'password' with 'p455w0rd':

git filter-repo --replace-text <(echo "password==>p455w0rd")

To use the current version of the .mailmap file to update authors,

committers, and taggers throughout history and make it permanent:
git filter-repo --use-mailmap

To extract the history of 'src/', rename all files to have a new leading
directory 'my-module' (e.g. src/foo.java -> my-module/src/foo.java), and
add a 'my-module-' prefix to all tags:
git filter-repo --path src/ --to-subdirectory-filter my-module --tag-rename
'':'my-module-'

For more detailed examples and explanations, see

https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/
blob/docs/html/git-filter-repo.html#EXAMPLES''')

# Create the basic parser

parser = argparse.ArgumentParser(description=summary,
usage = argparse.SUPPRESS,
add_help = False,
epilog = example_text,

formatter_class=argparse.RawDescriptionHelpFormatter)

analyze = parser.add_argument_group(title=_("Analysis"))
analyze.add_argument('--analyze', action='store_true',
help=_("Analyze repository history and create a report that may be "
"useful in determining what to filter in a subsequent run. "
"Will not modify your repo."))
analyze.add_argument('--report-dir',
metavar='DIR_OR_FILE',
type=os.fsencode,
dest='report_dir',
help=_("Directory to write report, defaults to
GIT_DIR/filter_repo/analysis,"
"refuses to run if exists, --force delete existing dir first."))

path = parser.add_argument_group(title=_("Filtering based on paths "

 "(see also --filename-callback)"),
description=textwrap.dedent(_("""
These options specify the paths to select. Note that much like git
itself, renames are NOT followed so you may need to specify multiple
paths, e.g. `--path olddir/ --path newdir/`
"""[1:])))

path.add_argument('--invert-paths', action='store_false', dest='inclusive',

help=_("Invert the selection of files from the specified "
"--path-{match,glob,regex} options below, i.e. only select "
"files matching none of those options."))

path.add_argument('--path-match', '--path', metavar='DIR_OR_FILE',

type=os.fsencode,
action=FilteringOptions.AppendFilter, dest='path_changes',
help=_("Exact paths (files or directories) to include in filtered "
"history. Multiple --path options can be specified to get "
"a union of paths."))
path.add_argument('--path-glob', metavar='GLOB', type=os.fsencode,
action=FilteringOptions.AppendFilter, dest='path_changes',
help=_("Glob of paths to include in filtered history. Multiple "
"--path-glob options can be specified to get a union of "
"paths."))
path.add_argument('--path-regex', metavar='REGEX', type=os.fsencode,
action=FilteringOptions.AppendFilter, dest='path_changes',
help=_("Regex of paths to include in filtered history. Multiple "
"--path-regex options can be specified to get a union of "
"paths"))
path.add_argument('--use-base-name', action='store_true',
help=_("Match on file base name instead of full path from the top "
"of the repo. Incompatible with --path-rename, and "
"incompatible with matching against directory names."))

rename = parser.add_argument_group(title=_("Renaming based on paths "

"(see also --filename-callback)"))
rename.add_argument('--path-rename', '--path-rename-match',
metavar='OLD_NAME:NEW_NAME', dest='path_changes', type=os.fsencode,
action=FilteringOptions.AppendFilter,
help=_("Path to rename; if filename or directory matches OLD_NAME "
"rename to NEW_NAME. Multiple --path-rename options can be "
"specified. NOTE: If you combine filtering options with "
"renaming ones, do not rely on a rename argument to select "
"paths; you also need a filter to select them."))

helpers = parser.add_argument_group(title=_("Path shortcuts"))

helpers.add_argument('--paths', help=argparse.SUPPRESS, metavar='IGNORE')
helpers.add_argument('--paths-from-file', metavar='FILENAME',
type=os.fsencode,
action=FilteringOptions.FileWithPathsFilter, dest='path_changes',
help=_("Specify several path filtering and renaming directives, one "
"per line. Lines with '==>' in them specify path renames, "
"and lines can begin with 'literal:' (the default), 'glob:', "
"or 'regex:' to specify different matching styles. Blank "
"lines and lines starting with a '#' are ignored."))
helpers.add_argument('--subdirectory-filter', metavar='DIRECTORY',
action=FilteringOptions.HelperFilter, type=os.fsencode,
help=_("Only look at history that touches the given subdirectory "
"and treat that directory as the project root. Equivalent "
"to using '--path DIRECTORY/ --path-rename DIRECTORY/:'"))
helpers.add_argument('--to-subdirectory-filter', metavar='DIRECTORY',
action=FilteringOptions.HelperFilter, type=os.fsencode,
help=_("Treat the project root as if it were under DIRECTORY. "
"Equivalent to using '--path-rename :DIRECTORY/'"))

contents = parser.add_argument_group(title=_("Content editing filters "

"(see also --blob-callback)"))
contents.add_argument('--replace-text', metavar='EXPRESSIONS_FILE',
help=_("A file with expressions that, if found, will be replaced. "
"By default, each expression is treated as literal text, "
"but 'regex:' and 'glob:' prefixes are supported. You can "
"end the line with '==>' and some replacement text to "
"choose a replacement choice other than the default of '{}'."
.format(decode(FilteringOptions.default_replace_text))))
contents.add_argument('--strip-blobs-bigger-than', metavar='SIZE',
dest='max_blob_size', default=0,
help=_("Strip blobs (files) bigger than specified size (e.g. '5M', "
"'2G', etc)"))
contents.add_argument('--strip-blobs-with-ids', metavar='BLOB-ID-FILENAME',
help=_("Read git object ids from each line of the given file, and "
"strip all of them from history"))

refrename = parser.add_argument_group(title=_("Renaming of refs "

"(see also --refname-callback)"))
refrename.add_argument('--tag-rename', metavar='OLD:NEW', type=os.fsencode,
help=_("Rename tags starting with OLD to start with NEW. For "
"example, --tag-rename foo:bar will rename tag foo-1.2.3 "
"to bar-1.2.3; either OLD or NEW can be empty."))

messages = parser.add_argument_group(title=_("Filtering of commit messages "

"(see also --message-callback)"))
messages.add_argument('--replace-message', metavar='EXPRESSIONS_FILE',
help=_("A file with expressions that, if found in commit or tag "
"messages, will be replaced. This file uses the same syntax "
"as --replace-text."))
messages.add_argument('--preserve-commit-hashes', action='store_true',
help=_("By default, since commits are rewritten and thus gain new "
"hashes, references to old commit hashes in commit messages "
"are replaced with new commit hashes (abbreviated to the same "
"length as the old reference). Use this flag to turn off "
"updating commit hashes in commit messages."))
messages.add_argument('--preserve-commit-encoding', action='store_true',
help=_("Do not reencode commit messages into UTF-8. By default, if "
"the commit object specifies an encoding for the commit "
"message, the message is re-encoded into UTF-8."))

people = parser.add_argument_group(title=_("Filtering of names & emails "

"(see also --name-callback "
"and --email-callback)"))
people.add_argument('--mailmap', dest='mailmap', metavar='FILENAME',
type=os.fsencode,
help=_("Use specified mailmap file (see git-shortlog(1) for "
"details on the format) when rewriting author, committer, "
"and tagger names and emails. If the specified file is "
"part of git history, historical versions of the file will "
"be ignored; only the current contents are consulted."))
people.add_argument('--use-mailmap', dest='mailmap',
action='store_const', const=b'.mailmap',
help=_("Same as: '--mailmap .mailmap' "))
parents = parser.add_argument_group(title=_("Parent rewriting"))
parents.add_argument('--replace-refs', default=None,
choices=['delete-no-add', 'delete-and-add',
'update-no-add', 'update-or-add',
'update-and-add', 'old-default'],
help=_("How to handle replace refs (see git-replace(1)). Replace "
"refs can be added during the history rewrite as a way to "
"allow users to pass old commit IDs (from before "
"git-filter-repo was run) to git commands and have git know "
"how to translate those old commit IDs to the new "
"(post-rewrite) commit IDs. Also, replace refs that existed "
"before the rewrite can either be deleted or updated. The "
"choices to pass to --replace-refs thus need to specify both "
"what to do with existing refs and what to do with commit "
"rewrites. Thus 'update-and-add' means to update existing "
"replace refs, and for any commit rewrite (even if already "
"pointed at by a replace ref) add a new refs/replace/ reference "
"to map from the old commit ID to the new commit ID. The "
"default is update-no-add, meaning update existing replace refs "
"but do not add any new ones. There is also a special "
"'old-default' option for picking the default used in versions "
"prior to git-filter-repo-2.45, namely 'update-and-add' upon "
"the first run of git-filter-repo in a repository and "
"'update-or-add' if running git-filter-repo again on a "
"repository."))
parents.add_argument('--prune-empty', default='auto',
choices=['always', 'auto', 'never'],
help=_("Whether to prune empty commits. 'auto' (the default) means "
"only prune commits which become empty (not commits which were "
"empty in the original repo, unless their parent was pruned). "
"When the parent of a commit is pruned, the first non-pruned "
"ancestor becomes the new parent."))
parents.add_argument('--prune-degenerate', default='auto',
choices=['always', 'auto', 'never'],
help=_("Since merge commits are needed for history topology, they "
"are typically exempt from pruning. However, they can become "
"degenerate with the pruning of other commits (having fewer "
"than two parents, having one commit serve as both parents, or "
"having one parent as the ancestor of the other.) If such "
"merge commits have no file changes, they can be pruned. The "
"default ('auto') is to only prune empty merge commits which "
"become degenerate (not which started as such)."))
parents.add_argument('--no-ff', action='store_true',
help=_("Even if the first parent is or becomes an ancestor of another "
"parent, do not prune it. This modifies how "
"--prune-degenerate behaves, and may be useful in projects who "
"always use merge --no-ff."))

callback = parser.add_argument_group(title=_("Generic callback code snippets"))

callback.add_argument('--filename-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing filenames; see CALLBACKS "
"sections below."))
callback.add_argument('--message-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing messages (both commit "
"messages and tag messages); see CALLBACKS section below."))
callback.add_argument('--name-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing names of people; see "
"CALLBACKS section below."))
callback.add_argument('--email-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing emails addresses; see "
"CALLBACKS section below."))
callback.add_argument('--refname-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing refnames; see CALLBACKS "
"section below."))

callback.add_argument('--blob-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing blob objects; see "
"CALLBACKS section below."))
callback.add_argument('--commit-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing commit objects; see "
"CALLBACKS section below."))
callback.add_argument('--tag-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing tag objects. Note that "
"lightweight tags have no tag object and are thus not "
"handled by this callback. See CALLBACKS section below."))
callback.add_argument('--reset-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing reset objects; see "
"CALLBACKS section below."))

desc = _(
"Specifying alternate source or target locations implies --partial,\n"
"except that the normal default for --replace-refs is used. However,\n"
"unlike normal uses of --partial, this doesn't risk mixing old and new\n"
"history since the old and new histories are in different repositories.")
location = parser.add_argument_group(title=_("Location to filter from/to"),
description=desc)
location.add_argument('--source', type=os.fsencode,
help=_("Git repository to read from"))
location.add_argument('--target', type=os.fsencode,
help=_("Git repository to overwrite with filtered history"))

order = parser.add_argument_group(title=_("Ordering of commits"))

order.add_argument('--date-order', action='store_true',
help=_("Processes commits in commit timestamp order."))

misc = parser.add_argument_group(title=_("Miscellaneous options"))

misc.add_argument('--help', '-h', action='store_true',
help=_("Show this help message and exit."))
misc.add_argument('--version', action='store_true',
help=_("Display filter-repo's version and exit."))
misc.add_argument('--proceed', action='store_true',
help=_("Avoid triggering the no-arguments-specified check."))
misc.add_argument('--force', '-f', action='store_true',
help=_("Rewrite repository history even if the current repo does not "
"look like a fresh clone. History rewriting is irreversible "
"(and includes immediate pruning of reflogs and old objects), "
"so be cautious about using this flag."))
misc.add_argument('--partial', action='store_true',
help=_("Do a partial history rewrite, resulting in the mixture of "
"old and new history. This disables rewriting "
"refs/remotes/origin/* to refs/heads/*, disables removing "
"of the 'origin' remote, disables removing unexported refs, "
"disables expiring the reflog, and disables the automatic "
"post-filter gc. Also, this modifies --tag-rename and "
"--refname-callback options such that instead of replacing "
"old refs with new refnames, it will instead create new "
"refs and keep the old ones around. Use with caution."))
 misc.add_argument('--no-gc', action='store_true',
help=_("Do not run 'git gc' after filtering."))
# WARNING: --refs presents a problem with become-degenerate pruning:
# * Excluding a commit also excludes its ancestors so when some other
# commit has an excluded ancestor as a parent we have no way of
# knowing what it is an ancestor of without doing a special
# full-graph walk.
misc.add_argument('--refs', nargs='+',
help=_("Limit history rewriting to the specified refs. Implies "
"--partial. In addition to the normal caveats of --partial "
"(mixing old and new history, no automatic remapping of "
"refs/remotes/origin/* to refs/heads/*, etc.), this also may "
"cause problems for pruning of degenerate empty merge "
"commits when negative revisions are specified."))

misc.add_argument('--dry-run', action='store_true',
help=_("Do not change the repository. Run `git fast-export` and "
"filter its output, and save both the original and the "
"filtered version for comparison. This also disables "
"rewriting commit messages due to not knowing new commit "
"IDs and disables filtering of some empty commits due to "
"inability to query the fast-import backend." ))
misc.add_argument('--debug', action='store_true',
help=_("Print additional information about operations being "
"performed and commands being run. When used together "
"with --dry-run, also show extra information about what "
"would be run."))
# WARNING: --state-branch has some problems:
# * It does not work well with manually inserted objects (user creating
# Blob() or Commit() or Tag() objects and calling
# RepoFilter.insert(obj) on them).
# * It does not work well with multiple source or multiple target repos
# * It doesn't work so well with pruning become-empty commits (though
# --refs doesn't work so well with it either)
# These are probably fixable, given some work (e.g. re-importing the
# graph at the beginning to get the AncestryGraph right, doing our own
# export of marks instead of using fast-export --export-marks, etc.), but
# for now just hide the option.
misc.add_argument('--state-branch',
#help=_("Enable incremental filtering by saving the mapping of old "
# "to new objects to the specified branch upon exit, and"
# "loading that mapping from that branch (if it exists) "
# "upon startup."))
help=argparse.SUPPRESS)
misc.add_argument('--stdin', action='store_true',
help=_("Instead of running `git fast-export` and filtering its "
"output, filter the fast-export stream from stdin. The "
"stdin must be in the expected input format (e.g. it needs "
"to include original-oid directives)."))
misc.add_argument('--quiet', action='store_true',
help=_("Pass --quiet to other git commands called"))
return parser

@staticmethod
def sanity_check_args(args):
if args.analyze and args.path_changes:
raise SystemExit(_("Error: --analyze is incompatible with --path* flags; "
"it's a read-only operation."))
if args.analyze and args.stdin:
 raise SystemExit(_("Error: --analyze is incompatible with --stdin."))
# If no path_changes are found, initialize with empty list but mark as
# not inclusive so that all files match
if args.path_changes == None:
args.path_changes = []
args.inclusive = False
else:
# Similarly, if we have no filtering paths, then no path should be
# filtered out. Based on how newname() works, the easiest way to
# achieve that is setting args.inclusive to False.
if not any(x[0] == 'filter' for x in args.path_changes):
args.inclusive = False
# Also check for incompatible --use-base-name and --path-rename flags.
if args.use_base_name:
if any(x[0] == 'rename' for x in args.path_changes):
raise SystemExit(_("Error: --use-base-name and --path-rename are "
"incompatible."))
# Also throw some sanity checks on git version here;
# PERF: remove these checks once new enough git versions are common
p = subproc.Popen('git fast-export -h'.split(),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.stdout.read()
if b'--anonymize-map' not in output: # pragma: no cover
global date_format_permissive
date_format_permissive = False
if not any(x in output for x in [b'--mark-tags',b'--[no-]mark-tags']): #
pragma: no cover
global write_marks
write_marks = False
if args.state_branch:
# We need a version of git-fast-export with --mark-tags
raise SystemExit(_("Error: need git >= 2.24.0"))
if not any(x in output for x in [b'--reencode', b'--[no-]reencode']): #
pragma: no cover
if args.preserve_commit_encoding:
# We need a version of git-fast-export with --reencode
raise SystemExit(_("Error: need git >= 2.23.0"))
else:
# Set args.preserve_commit_encoding to None which we'll check for later
# to avoid passing --reencode=yes to fast-export (that option was the
# default prior to git-2.23)
args.preserve_commit_encoding = None
# If we don't have fast-exoprt --reencode, we may also be missing
# diff-tree --combined-all-paths, which is even more important...
p = subproc.Popen('git diff-tree -h'.split(),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.stdout.read()
if b'--combined-all-paths' not in output:
# We need a version of git-diff-tree with --combined-all-paths
raise SystemExit(_("Error: need git >= 2.22.0"))
# End of sanity checks on git version
if args.max_blob_size:
suffix = args.max_blob_size[-1]
if suffix not in '1234567890':
mult = {'K': 1024, 'M': 1024**2, 'G': 1024**3}
if suffix not in mult:
raise SystemExit(_("Error: could not parse --strip-blobs-bigger-than"
" argument %s")
% args.max_blob_size)
 args.max_blob_size = int(args.max_blob_size[0:-1]) * mult[suffix]
else:
args.max_blob_size = int(args.max_blob_size)

@staticmethod
def get_replace_text(filename):
replace_literals = []
replace_regexes = []
with open(filename, 'br') as f:
for line in f:
line = line.rstrip(b'\r\n')

# Determine the replacement

replacement = FilteringOptions.default_replace_text
if b'==>' in line:
line, replacement = line.rsplit(b'==>', 1)

# See if we need to match via regex

regex = None
if line.startswith(b'regex:'):
regex = line[6:]
elif line.startswith(b'glob:'):
regex = glob_to_regex(line[5:])
if regex:
replace_regexes.append((re.compile(regex), replacement))
else:
# Otherwise, find the literal we need to replace
if line.startswith(b'literal:'):
line = line[8:]
if not line:
continue
replace_literals.append((line, replacement))
return {'literals': replace_literals, 'regexes': replace_regexes}

@staticmethod
def get_paths_from_file(filename):
new_path_changes = []
with open(filename, 'br') as f:
for line in f:
line = line.rstrip(b'\r\n')

# Skip blank lines

if not line:
continue
# Skip comment lines
if line.startswith(b'#'):
continue

# Determine the replacement

match_type, repl = 'literal', None
if b'==>' in line:
line, repl = line.rsplit(b'==>', 1)

# See if we need to match via regex

match_type = 'match' # a.k.a. 'literal'
if line.startswith(b'regex:'):
match_type = 'regex'
match = re.compile(line[6:])
elif line.startswith(b'glob:'):
 match_type = 'glob'
match = line[5:]
if repl:
raise SystemExit(_("Error: In %s, 'glob:' and '==>' are incompatible
(renaming globs makes no sense)" % decode(filename)))
else:
if line.startswith(b'literal:'):
match = line[8:]
else:
match = line
if repl is not None:
if match and repl and match.endswith(b'/') != repl.endswith(b'/'):
raise SystemExit(_("Error: When rename directories, if OLDNAME "
"and NEW_NAME are both non-empty and either "
"ends with a slash then both must."))

# Record the filter or rename

if repl is not None:
new_path_changes.append(['rename', match_type, (match, repl)])
else:
new_path_changes.append(['filter', match_type, match])
if match_type == 'glob' and not match.endswith(b'*'):
extension = b'*' if match.endswith(b'/') else b'/*'
new_path_changes.append(['filter', match_type, match+extension])
return new_path_changes

@staticmethod
def default_options():
return FilteringOptions.parse_args([], error_on_empty = False)

@staticmethod
def parse_args(input_args, error_on_empty = True):
parser = FilteringOptions.create_arg_parser()
if not input_args and error_on_empty:
parser.print_usage()
raise SystemExit(_("No arguments specified."))
args = parser.parse_args(input_args)
if args.help:
parser.print_help()
raise SystemExit()
if args.paths:
raise SystemExit("Error: Option `--paths` unrecognized; did you mean --path
or --paths-from-file?")
if args.version:
GitUtils.print_my_version()
raise SystemExit()
FilteringOptions.sanity_check_args(args)
if args.mailmap:
args.mailmap = MailmapInfo(args.mailmap)
if args.replace_text:
args.replace_text = FilteringOptions.get_replace_text(args.replace_text)
if args.replace_message:
args.replace_message =
FilteringOptions.get_replace_text(args.replace_message)
if args.strip_blobs_with_ids:
with open(args.strip_blobs_with_ids, 'br') as f:
args.strip_blobs_with_ids = set(f.read().split())
else:
args.strip_blobs_with_ids = set()
 if (args.partial or args.refs) and not args.replace_refs:
args.replace_refs = 'update-no-add'
args.repack = not (args.partial or args.refs or args.no_gc)
if args.refs or args.source or args.target:
args.partial = True
if not args.refs:
args.refs = ['--all']
return args

class RepoAnalyze(object):

# First, several helper functions for analyze_commit()

@staticmethod
def equiv_class(stats, filename):
return stats['equivalence'].get(filename, (filename,))

@staticmethod
def setup_equivalence_for_rename(stats, oldname, newname):
# if A is renamed to B and B is renamed to C, then the user thinks of
# A, B, and C as all being different names for the same 'file'. We record
# this as an equivalence class:
# stats['equivalence'][name] = (A,B,C)
# for name being each of A, B, and C.
old_tuple = stats['equivalence'].get(oldname, ())
if newname in old_tuple:
return
elif old_tuple:
new_tuple = tuple(list(old_tuple)+[newname])
else:
new_tuple = (oldname, newname)
for f in new_tuple:
stats['equivalence'][f] = new_tuple

@staticmethod
def setup_or_update_rename_history(stats, commit, oldname, newname):
rename_commits = stats['rename_history'].get(oldname, set())
rename_commits.add(commit)
stats['rename_history'][oldname] = rename_commits

@staticmethod
def handle_renames(stats, commit, change_types, filenames):
for index, change_type in enumerate(change_types):
if change_type == ord(b'R'):
oldname, newname = filenames[index], filenames[-1]
RepoAnalyze.setup_equivalence_for_rename(stats, oldname, newname)
RepoAnalyze.setup_or_update_rename_history(stats, commit,
oldname, newname)

@staticmethod
def handle_file(stats, graph, commit, modes, shas, filenames):
mode, sha, filename = modes[-1], shas[-1], filenames[-1]

# Figure out kind of deletions to undo for this file, and update lists
# of all-names-by-sha and all-filenames
delmode = 'tree_deletions'
if mode != b'040000':
delmode = 'file_deletions'
stats['names'][sha].add(filename)
 stats['allnames'].add(filename)

# If the file (or equivalence class of files) was recorded as deleted,

# clearly it isn't anymore
equiv = RepoAnalyze.equiv_class(stats, filename)
for f in equiv:
stats[delmode].pop(f, None)

# If we get a modify/add for a path that was renamed, we may need to break
# the equivalence class. However, if the modify/add was on a branch that
# doesn't have the rename in its history, we are still okay.
need_to_break_equivalence = False
if equiv[-1] != filename:
for rename_commit in stats['rename_history'][filename]:
if graph.is_ancestor(rename_commit, commit):
need_to_break_equivalence = True

if need_to_break_equivalence:
for f in equiv:
if f in stats['equivalence']:
del stats['equivalence'][f]

@staticmethod
def analyze_commit(stats, graph, commit, parents, date, file_changes):
graph.add_commit_and_parents(commit, parents)
for change in file_changes:
modes, shas, change_types, filenames = change
if len(parents) == 1 and change_types.startswith(b'R'):
change_types = b'R' # remove the rename score; we don't care
if modes[-1] == b'160000':
continue
elif modes[-1] == b'000000':
# Track when files/directories are deleted
for f in RepoAnalyze.equiv_class(stats, filenames[-1]):
if any(x == b'040000' for x in modes[0:-1]):
stats['tree_deletions'][f] = date
else:
stats['file_deletions'][f] = date
elif change_types.strip(b'AMT') == b'':
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
elif modes[-1] == b'040000' and change_types.strip(b'RAM') == b'':
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
elif change_types.strip(b'RAMT') == b'':
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
RepoAnalyze.handle_renames(stats, commit, change_types, filenames)
else:
raise SystemExit(_("Unhandled change type(s): %(change_type)s "
"(in commit %(commit)s)")
% ({'change_type': change_types, 'commit': commit})
) # pragma: no cover

@staticmethod
def gather_data(args):
unpacked_size, packed_size = GitUtils.get_blob_sizes()
stats = {'names': collections.defaultdict(set),
'allnames' : set(),
'file_deletions': {},
'tree_deletions': {},
'equivalence': {},
 'rename_history': collections.defaultdict(set),
'unpacked_size': unpacked_size,
'packed_size': packed_size,
'num_commits': 0}

# Setup the rev-list/diff-tree process

processed_commits_msg = _("Processed %d commits")
commit_parse_progress = ProgressWriter()
num_commits = 0
cmd = ('git rev-list --topo-order --reverse {}'.format(' '.join(args.refs)) +
' | git diff-tree --stdin --always --root --format=%H%n%P%n%cd' +
' --date=short -M -t -c --raw --combined-all-paths')
dtp = subproc.Popen(cmd, shell=True, bufsize=-1, stdout=subprocess.PIPE)
f = dtp.stdout
line = f.readline()
if not line:
raise SystemExit(_("Nothing to analyze; repository is empty."))
cont = bool(line)
graph = AncestryGraph()
while cont:
commit = line.rstrip()
parents = f.readline().split()
date = f.readline().rstrip()

# We expect a blank line next; if we get a non-blank line then

# this commit modified no files and we need to move on to the next.
# If there is no line, we've reached end-of-input.
line = f.readline()
if not line:
cont = False
line = line.rstrip()

# If we haven't reached end of input, and we got a blank line meaning

# a commit that has modified files, then get the file changes associated
# with this commit.
file_changes = []
if cont and not line:
cont = False
for line in f:
if not line.startswith(b':'):
cont = True
break
n = 1+max(1, len(parents))
assert line.startswith(b':'*(n-1))
relevant = line[n-1:-1]
splits = relevant.split(None, n)
modes = splits[0:n]
splits = splits[n].split(None, n)
shas = splits[0:n]
splits = splits[n].split(b'\t')
change_types = splits[0]
filenames = [PathQuoting.dequote(x) for x in splits[1:]]
file_changes.append([modes, shas, change_types, filenames])

# If someone is trying to analyze a subset of the history, make sure

# to avoid dying on commits with parents that we haven't seen before
if args.refs:
graph.record_external_commits([p for p in parents
if not p in graph.value])
 # Analyze this commit and update progress
RepoAnalyze.analyze_commit(stats, graph, commit, parents, date,
file_changes)
num_commits += 1
commit_parse_progress.show(processed_commits_msg % num_commits)

# Show the final commits processed message and record the number of commits
commit_parse_progress.finish()
stats['num_commits'] = num_commits

# Close the output, ensure rev-list|diff-tree pipeline completed successfully

dtp.stdout.close()
if dtp.wait():
raise SystemExit(_("Error: rev-list|diff-tree pipeline failed; see above."))
# pragma: no cover

return stats

@staticmethod
def write_report(reportdir, stats):
def datestr(datetimestr):
return datetimestr if datetimestr else _('<present>').encode()

def dirnames(path):
while True:
path = os.path.dirname(path)
yield path
if path == b'':
break

# Compute aggregate size information for paths, extensions, and dirs

total_size = {'packed': 0, 'unpacked': 0}
path_size = {'packed': collections.defaultdict(int),
'unpacked': collections.defaultdict(int)}
ext_size = {'packed': collections.defaultdict(int),
'unpacked': collections.defaultdict(int)}
dir_size = {'packed': collections.defaultdict(int),
'unpacked': collections.defaultdict(int)}
for sha in stats['names']:
size = {'packed': stats['packed_size'][sha],
'unpacked': stats['unpacked_size'][sha]}
for which in ('packed', 'unpacked'):
for name in stats['names'][sha]:
total_size[which] += size[which]
path_size[which][name] += size[which]
basename, ext = os.path.splitext(name)
ext_size[which][ext] += size[which]
for dirname in dirnames(name):
dir_size[which][dirname] += size[which]

# Determine if and when extensions and directories were deleted

ext_deleted_data = {}
for name in stats['allnames']:
when = stats['file_deletions'].get(name, None)

# Update the extension

basename, ext = os.path.splitext(name)
if when is None:
 ext_deleted_data[ext] = None
elif ext in ext_deleted_data:
if ext_deleted_data[ext] is not None:
ext_deleted_data[ext] = max(ext_deleted_data[ext], when)
else:
ext_deleted_data[ext] = when

dir_deleted_data = {}
for name in dir_size['packed']:
dir_deleted_data[name] = stats['tree_deletions'].get(name, None)

with open(os.path.join(reportdir, b"README"), 'bw') as f:

# Give a basic overview of this file
f.write(b"== %s ==\n" % _("Overall Statistics").encode())
f.write((" %s: %d\n" % (_("Number of commits"),
stats['num_commits'])).encode())
f.write((" %s: %d\n" % (_("Number of filenames"),
len(path_size['packed']))).encode())
f.write((" %s: %d\n" % (_("Number of directories"),
len(dir_size['packed']))).encode())
f.write((" %s: %d\n" % (_("Number of file extensions"),
len(ext_size['packed']))).encode())
f.write(b"\n")
f.write((" %s: %d\n" % (_("Total unpacked size (bytes)"),
total_size['unpacked'])).encode())
f.write((" %s: %d\n" % (_("Total packed size (bytes)"),
total_size['packed'])).encode())
f.write(b"\n")

# Mention issues with the report

f.write(("== %s ==\n" % _("Caveats")).encode())
f.write(("=== %s ===\n" % _("Sizes")).encode())
f.write(textwrap.dedent(_("""
Packed size represents what size your repository would be if no
trees, commits, tags, or other metadata were included (though it may
fail to represent de-duplication; see below). It also represents the
current packing, which may be suboptimal if you haven't gc'ed for a
while.

Unpacked size represents what size your repository would be if no

trees, commits, tags, or other metadata were included AND if no
files were packed; i.e., without delta-ing or compression.

Both unpacked and packed sizes can be slightly misleading. Deleting

a blob from history not save as much space as the unpacked size,
because it is obviously normally stored in packed form. Also,
deleting a blob from history may not save as much space as its packed
size either, because another blob could be stored as a delta against
that blob, so when you remove one blob another blob's packed size may
grow.

Also, the sum of the packed sizes can add up to more than the
repository size; if the same contents appeared in the repository in
multiple places, git will automatically de-dupe and store only one
copy, while the way sizes are added in this analysis adds the size
for each file path that has those contents. Further, if a file is
ever reverted to a previous version's contents, the previous
version's size will be counted multiple times in this analysis, even
though git will only store it once.
 """)[1:]).encode())
f.write(b"\n")
f.write(("=== %s ===\n" % _("Deletions")).encode())
f.write(textwrap.dedent(_("""
Whether a file is deleted is not a binary quality, since it can be
deleted on some branches but still exist in others. Also, it might
exist in an old tag, but have been deleted in versions newer than
that. More thorough tracking could be done, including looking at
merge commits where one side of history deleted and the other modified,
in order to give a more holistic picture of deletions. However, that
algorithm would not only be more complex to implement, it'd also be
quite difficult to present and interpret by users. Since --analyze
is just about getting a high-level rough picture of history, it instead
implements the simplistic rule that is good enough for 98% of cases:
A file is marked as deleted if the last commit in the fast-export
stream that mentions the file lists it as deleted.
This makes it dependent on topological ordering, but generally gives
the "right" answer.
""")[1:]).encode())
f.write(b"\n")
f.write(("=== %s ===\n" % _("Renames")).encode())
f.write(textwrap.dedent(_("""
Renames share the same non-binary nature that deletions do, plus
additional challenges:
* If the renamed file is renamed again, instead of just two names for
a path you can have three or more.
* Rename pairs of the form (oldname, newname) that we consider to be
different names of the "same file" might only be valid over certain
commit ranges. For example, if a new commit reintroduces a file
named oldname, then new versions of oldname aren't the "same file"
anymore. We could try to portray this to the user, but it's easier
for the user to just break the pairing and only report unbroken
rename pairings to the user.
* The ability for users to rename files differently in different
branches means that our chains of renames will not necessarily be
linear but may branch out.
""")[1:]).encode())
f.write(b"\n")

# Equivalence classes for names, so if folks only want to keep a

# certain set of paths, they know the old names they want to include
# too.
with open(os.path.join(reportdir, b"renames.txt"), 'bw') as f:
seen = set()
for pathname,equiv_group in sorted(stats['equivalence'].items(),
key=lambda x:(x[1], x[0])):
if equiv_group in seen:
continue
seen.add(equiv_group)
f.write(("{} ->\n ".format(decode(equiv_group[0])) +
"\n ".join(decode(x) for x in equiv_group[1:]) +
"\n").encode())

# List directories in reverse sorted order of unpacked size

with open(os.path.join(reportdir, b"directories-deleted-sizes.txt"), 'bw') as
f:
msg = "=== %s ===\n" % _("Deleted directories by reverse size")
f.write(msg.encode())
msg = _("Format: unpacked size, packed size, date deleted, directory name\n")
 f.write(msg.encode())
for dirname, size in sorted(dir_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
if (dir_deleted_data[dirname]):
f.write(b" %10d %10d %-10s %s\n" % (dir_size['unpacked'][dirname],
size,
datestr(dir_deleted_data[dirname]),
dirname or _('<toplevel>').encode()))

with open(os.path.join(reportdir, b"directories-all-sizes.txt"), 'bw') as f:

f.write(("=== %s ===\n" % _("All directories by reverse size")).encode())
msg = _("Format: unpacked size, packed size, date deleted, directory name\n")
f.write(msg.encode())
for dirname, size in sorted(dir_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
f.write(b" %10d %10d %-10s %s\n" % (dir_size['unpacked'][dirname],
size,
datestr(dir_deleted_data[dirname]),
dirname or _("<toplevel>").encode()))

# List extensions in reverse sorted order of unpacked size

with open(os.path.join(reportdir, b"extensions-deleted-sizes.txt"), 'bw') as f:
msg = "=== %s ===\n" % _("Deleted extensions by reverse size")
f.write(msg.encode())
msg = _("Format: unpacked size, packed size, date deleted, extension name\n")
f.write(msg.encode())
for extname, size in sorted(ext_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
if (ext_deleted_data[extname]):
f.write(b" %10d %10d %-10s %s\n" % (ext_size['unpacked'][extname],
size,
datestr(ext_deleted_data[extname]),
extname or _('<no
extension>').encode()))

with open(os.path.join(reportdir, b"extensions-all-sizes.txt"), 'bw') as f:

f.write(("=== %s ===\n" % _("All extensions by reverse size")).encode())
msg = _("Format: unpacked size, packed size, date deleted, extension name\n")
f.write(msg.encode())
for extname, size in sorted(ext_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
f.write(b" %10d %10d %-10s %s\n" % (ext_size['unpacked'][extname],
size,
datestr(ext_deleted_data[extname]),
extname or _('<no
extension>').encode()))

# List files in reverse sorted order of unpacked size

with open(os.path.join(reportdir, b"path-deleted-sizes.txt"), 'bw') as f:
msg = "=== %s ===\n" % _("Deleted paths by reverse accumulated size")
f.write(msg.encode())
msg = _("Format: unpacked size, packed size, date deleted, path name(s)\n")
f.write(msg.encode())
for pathname, size in sorted(path_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
when = stats['file_deletions'].get(pathname, None)
if when:
f.write(b" %10d %10d %-10s %s\n" % (path_size['unpacked'][pathname],
size,
 datestr(when),
pathname))

with open(os.path.join(reportdir, b"path-all-sizes.txt"), 'bw') as f:

msg = "=== %s ===\n" % _("All paths by reverse accumulated size")
f.write(msg.encode())
msg = _("Format: unpacked size, packed size, date deleted, path name\n")
f.write(msg.encode())
for pathname, size in sorted(path_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
when = stats['file_deletions'].get(pathname, None)
f.write(b" %10d %10d %-10s %s\n" % (path_size['unpacked'][pathname],
size,
datestr(when),
pathname))

# List of filenames and sizes in descending order

with open(os.path.join(reportdir, b"blob-shas-and-paths.txt"), 'bw') as f:
f.write(("=== %s ===\n" % _("Files by sha and associated pathnames in reverse
size")).encode())
f.write(_("Format: sha, unpacked size, packed size, filename(s) object stored
as\n").encode())
for sha, size in sorted(stats['packed_size'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
if sha not in stats['names']:
# Some objects in the repository might not be referenced, or not
# referenced by the branches/tags the user cares about; skip them.
continue
names_with_sha = stats['names'][sha]
if len(names_with_sha) == 1:
names_with_sha = names_with_sha.pop()
else:
names_with_sha = b'[' + b', '.join(sorted(names_with_sha)) + b']'
f.write(b" %s %10d %10d %s\n" % (sha,
stats['unpacked_size'][sha],
size,
names_with_sha))

@staticmethod
def run(args):
if args.report_dir:
reportdir = args.report_dir
else:
git_dir = GitUtils.determine_git_dir(b'.')

# Create the report directory as necessary

results_tmp_dir = os.path.join(git_dir, b'filter-repo')
if not os.path.isdir(results_tmp_dir):
os.mkdir(results_tmp_dir)
reportdir = os.path.join(results_tmp_dir, b"analysis")

if os.path.isdir(reportdir):
if args.force:
sys.stdout.write(_("Warning: Removing recursively: \"%s\"") %
decode(reportdir))
shutil.rmtree(reportdir)
else:
sys.stdout.write(_("Error: dir already exists (use --force to
delete): \"%s\"\n") % decode(reportdir))
 sys.exit(1)

os.mkdir(reportdir)

# Gather the data we need

stats = RepoAnalyze.gather_data(args)

# Write the reports

sys.stdout.write(_("Writing reports to %s...") % decode(reportdir))
sys.stdout.flush()
RepoAnalyze.write_report(reportdir, stats)
sys.stdout.write(_("done.\n"))

class InputFileBackup:
def __init__(self, input_file, output_file):
self.input_file = input_file
self.output_file = output_file

def close(self):
self.input_file.close()
self.output_file.close()

def read(self, size):

output = self.input_file.read(size)
self.output_file.write(output)
return output

def readline(self):
line = self.input_file.readline()
self.output_file.write(line)
return line

class DualFileWriter:
def __init__(self, file1, file2):
self.file1 = file1
self.file2 = file2

def write(self, *args):

self.file1.write(*args)
self.file2.write(*args)

def flush(self):
self.file1.flush()
self.file2.flush()

def close(self):
self.file1.close()
self.file2.close()

class RepoFilter(object):
def __init__(self,
args,
filename_callback = None,
message_callback = None,
name_callback = None,
email_callback = None,
refname_callback = None,
blob_callback = None,
commit_callback = None,
 tag_callback = None,
reset_callback = None,
done_callback = None):

self._args = args

# Repo we are exporting

self._repo_working_dir = None

# Store callbacks for acting on objects printed by FastExport

self._blob_callback = blob_callback
self._commit_callback = commit_callback
self._tag_callback = tag_callback
self._reset_callback = reset_callback
self._done_callback = done_callback

# Store callbacks for acting on slices of FastExport objects

self._filename_callback = filename_callback # filenames from commits
self._message_callback = message_callback # commit OR tag message
self._name_callback = name_callback # author, committer, tagger
self._email_callback = email_callback # author, committer, tagger
self._refname_callback = refname_callback # from commit/tag/reset
self._handle_arg_callbacks()

# Defaults for input

self._input = None
self._fep = None # Fast Export Process
self._fe_orig = None # Path to where original fast-export output stored
self._fe_filt = None # Path to where filtered fast-export output stored
self._parser = None # FastExportParser object we are working with

# Defaults for output

self._output = None
self._fip = None # Fast Import Process
self._import_pipes = None
self._managed_output = True

# A tuple of (depth, list-of-ancestors). Commits and ancestors are

# identified by their id (their 'mark' in fast-export or fast-import
# speak). The depth of a commit is one more than the max depth of any
# of its ancestors.
self._graph = AncestryGraph()
# Another one, for ancestry of commits in the original repo
self._orig_graph = AncestryGraph()

# Names of files that were tweaked in any commit; such paths could lead
# to subsequent commits being empty
self._files_tweaked = set()

# A set of commit hash pairs (oldhash, newhash) which used to be merge

# commits but due to filtering were turned into non-merge commits.
# The commits probably have suboptimal commit messages (e.g. "Merge branch
# next into master").
self._commits_no_longer_merges = []

# A dict of original_ids to new_ids; filtering commits means getting

# new commit hash (sha1sums), and we record the mapping both for
# diagnostic purposes and so we can rewrite commit messages. Note that
# the new_id can be None rather than a commit hash if the original
 # commit became empty and was pruned or was otherwise dropped.
self._commit_renames = {}

# A set of original_ids (i.e. original hashes) for which we have not yet
# gotten the new hashses; the value is always the corresponding fast-export
# id (i.e. commit.id)
self._pending_renames = collections.OrderedDict()

# A dict of commit_hash[0:7] -> set(commit_hashes with that prefix).

#
# It's common for commit messages to refer to commits by abbreviated
# commit hashes, as short as 7 characters. To facilitate translating
# such short hashes, we have a mapping of prefixes to full old hashes.
self._commit_short_old_hashes = collections.defaultdict(set)

# A set of commit hash references appearing in commit messages which

# mapped to a valid commit that was removed entirely in the filtering
# process. The commit message will continue to reference the
# now-missing commit hash, since there was nothing to map it to.
self._commits_referenced_but_removed = set()

# Progress handling (number of commits parsed, etc.)

self._progress_writer = ProgressWriter()
self._num_commits = 0

# Size of blobs in the repo

self._unpacked_size = {}

# Other vars
self._sanity_checks_handled = False
self._finalize_handled = False
self._orig_refs = None
self._config_settings = {}
self._newnames = {}
self._stash = None

# Cache a few message translations for performance reasons

self._parsed_message = _("Parsed %d commits")

# Compile some regexes and cache those

self._hash_re = re.compile(br'(\b[0-9a-f]{7,40}\b)')
self._full_hash_re = re.compile(br'(\b[0-9a-f]{40}\b)')

def _handle_arg_callbacks(self):
def make_callback(argname, str):
callback_globals = {g: globals()[g] for g in public_globals}
callback_locals = {}
exec('def callback({}, _do_not_use_this_var = None):\n'.format(argname)+
' '+'\n '.join(str.splitlines()), callback_globals, callback_locals)
return callback_locals['callback']
def handle(type):
callback_field = '_{}_callback'.format(type)
code_string = getattr(self._args, type+'_callback')
if code_string:
if os.path.exists(code_string):
with open(code_string, 'r', encoding='utf-8') as f:
code_string = f.read()
if getattr(self, callback_field):
raise SystemExit(_("Error: Cannot pass a %s_callback to RepoFilter "
 "AND pass --%s-callback"
% (type, type)))
if 'return ' not in code_string and \
type not in ('blob', 'commit', 'tag', 'reset'):
raise SystemExit(_("Error: --%s-callback should have a return statement")
% type)
setattr(self, callback_field, make_callback(type, code_string))
handle('filename')
handle('message')
handle('name')
handle('email')
handle('refname')
handle('blob')
handle('commit')
handle('tag')
handle('reset')

def _run_sanity_checks(self):
self._sanity_checks_handled = True
if not self._managed_output:
if not self._args.replace_refs:
# If not _managed_output we don't want to make extra changes to the
# repo, so set default to no-op 'update-no-add'
self._args.replace_refs = 'update-no-add'
return

if self._args.debug:
print("[DEBUG] Passed arguments:\n{}".format(self._args))

# Determine basic repository information

target_working_dir = self._args.target or b'.'
self._orig_refs = GitUtils.get_refs(target_working_dir)
is_bare = GitUtils.is_repository_bare(target_working_dir)
self._config_settings = GitUtils.get_config_settings(target_working_dir)

# Determine if this is second or later run of filter-repo

tmp_dir = self.results_tmp_dir(create_if_missing=False)
ran_path = os.path.join(tmp_dir, b'already_ran')
already_ran = os.path.isfile(ran_path)
if already_ran:
current_time = time.time()
file_mod_time = os.path.getmtime(ran_path)
file_age = current_time - file_mod_time
if file_age > 86400: # file older than a day
msg = (f"The previous run is older than a day ({decode(ran_path)} already
exists).\n"
f"See \"Already Ran\" section in the manual for more information.\n"
f"Treat this run as a continuation of filtering in the previous run
(Y/N)? ")
response = input(msg)

if response.lower() != 'y':
os.remove(ran_path)
already_ran = False

# Default for --replace-refs

if not self._args.replace_refs:
self._args.replace_refs = 'delete-no-add'
if self._args.replace_refs == 'old-default':
 self._args.replace_refs = ('update-or-add' if already_ran
else 'update-and-add')

# Do sanity checks from the correct directory

if not self._args.force and not already_ran:
cwd = os.getcwd()
os.chdir(target_working_dir)
RepoFilter.sanity_check(self._orig_refs, is_bare, self._config_settings)
os.chdir(cwd)

@staticmethod
def loose_objects_are_replace_refs(git_dir, refs, num_loose_objects):
replace_objects = set()
for refname, rev in refs.items():
if not refname.startswith(b'refs/replace/'):
continue
replace_objects.add(rev)

validobj_re = re.compile(rb'^[0-9a-f]{40}$')
object_dir=os.path.join(git_dir, b'objects')
for root, dirs, files in os.walk(object_dir):
for filename in files:
objname = os.path.basename(root)+filename
if objname not in replace_objects and validobj_re.match(objname):
return False

return True

@staticmethod
def sanity_check(refs, is_bare, config_settings):
def abort(reason):
dirname = config_settings.get(b'remote.origin.url', b'')
msg = ""
if dirname and os.path.isdir(dirname):
msg = _("Note: when cloning local repositories, you need to pass\n"
" --no-local to git clone to avoid this issue.\n")
raise SystemExit(
_("Aborting: Refusing to destructively overwrite repo history since\n"
"this does not look like a fresh clone.\n"
" (%s)\n%s"
"Please operate on a fresh clone instead. If you want to proceed\n"
"anyway, use --force.") % (reason, msg))

# Avoid letting people running with weird setups and overwriting GIT_DIR
# elsewhere
git_dir = GitUtils.determine_git_dir(b'.')
if is_bare and git_dir != b'.':
abort(_("GIT_DIR must be ."))
elif not is_bare and git_dir != b'.git':
abort(_("GIT_DIR must be .git"))

# Check for refname collisions

if config_settings.get(b'core.ignorecase', b'false') == b'true':
collisions = collections.defaultdict(list)
for ref in refs:
collisions[ref.lower()].append(ref)
msg = ""
for ref in collisions:
if len(collisions[ref]) >= 2:
 msg += " " + decode(b", ".join(collisions[ref])) + "\n"
if msg:
raise SystemExit(
_("Aborting: Cannot rewrite history on a case insensitive\n"
"filesystem since you have refs that differ in case only:\n"
"%s") % msg)
if config_settings.get(b'core.precomposeunicode', b'false') == b'true':
import unicodedata # Mac users need to have python-3.8
collisions = collections.defaultdict(list)
for ref in refs:
strref = decode(ref)
collisions[unicodedata.normalize('NFC', strref)].append(strref)
msg = ""
for ref in collisions:
if len(collisions[ref]) >= 2:
msg += " " + ", ".join(collisions[ref]) + "\n"
if msg:
raise SystemExit(
_("Aborting: Cannot rewrite history on a character normalizing\n"
"filesystem since you have refs that differ in normalization:\n"
"%s") % msg)

# Make sure repo is fully packed, just like a fresh clone would be.
# Note that transfer.unpackLimit defaults to 100, meaning that a
# repository with no packs and less than 100 objects should be considered
# fully packed.
output = subproc.check_output('git count-objects -v'.split())
stats = dict(x.split(b': ') for x in output.splitlines())
num_packs = int(stats[b'packs'])
num_loose_objects = int(stats[b'count'])
if num_packs > 1 or \
num_loose_objects >= 100 or \
(num_packs == 1 and num_loose_objects > 0 and
not RepoFilter.loose_objects_are_replace_refs(git_dir, refs,
num_loose_objects)):
abort(_("expected freshly packed repo"))

# Make sure there is precisely one remote, named "origin"...or that this
# is a new bare repo with no packs and no remotes
output = subproc.check_output('git remote'.split()).strip()
if not (output == b"origin" or (num_packs == 0 and not output)):
abort(_("expected one remote, origin"))

# Make sure that all reflogs have precisely one entry

reflog_dir=os.path.join(git_dir, b'logs')
for root, dirs, files in os.walk(reflog_dir):
for filename in files:
pathname = os.path.join(root, filename)
with open(pathname, 'br') as f:
if len(f.read().splitlines()) > 1:
shortpath = pathname[len(reflog_dir)+1:]
abort(_("expected at most one entry in the reflog for %s") %
decode(shortpath))

# Make sure there are no stashed changes

if b'refs/stash' in refs:
abort(_("has stashed changes"))

# Do extra checks in non-bare repos

 if not is_bare:
# Avoid uncommitted, unstaged, or untracked changes
if subproc.call('git diff --staged --quiet'.split()):
abort(_("you have uncommitted changes"))
if subproc.call('git diff --quiet'.split()):
abort(_("you have unstaged changes"))
untracked_output = subproc.check_output('git ls-files -o'.split())
if len(untracked_output) > 0:
uf = untracked_output.rstrip(b'\n').split(b'\n')
# Since running git-filter-repo can result in files being written to
# __pycache__ (depending on python version, env vars, etc.), let's
# ignore those as far as "clean clone" is concerned.
relevant_uf = [x for x in uf
if not x.startswith(b'__pycache__/git_filter_repo.')]
if len(relevant_uf) > 0:
abort(_("you have untracked changes"))

# Avoid unpushed changes

for refname, rev in refs.items():
if not refname.startswith(b'refs/heads/'):
continue
origin_ref = refname.replace(b'refs/heads/', b'refs/remotes/origin/')
if origin_ref not in refs:
abort(_('%s exists, but %s not found') % (decode(refname),
decode(origin_ref)))
if rev != refs[origin_ref]:
abort(_('%s does not match %s') % (decode(refname),
decode(origin_ref)))

# Make sure there is only one worktree

output = subproc.check_output('git worktree list'.split())
if len(output.splitlines()) > 1:
abort(_('you have multiple worktrees'))

def cleanup(self, repo, repack, reset,

run_quietly=False, show_debuginfo=False):
''' cleanup repo; if repack then expire reflogs and do a gc --prune=now.
if reset then do a reset --hard. Optionally also curb output if
run_quietly is True, or go the opposite direction and show extra
output if show_debuginfo is True. '''
assert not (run_quietly and show_debuginfo)

if (repack and not run_quietly and not show_debuginfo):

print(_("Repacking your repo and cleaning out old unneeded objects"))
quiet_flags = '--quiet' if run_quietly else ''
cleanup_cmds = []
if repack:
cleanup_cmds = ['git reflog expire --expire=now --all'.split(),
'git gc {} --prune=now'.format(quiet_flags).split()]
if reset:
cleanup_cmds.insert(0, 'git reset {} --hard'.format(quiet_flags).split())
location_info = ' (in {})'.format(decode(repo)) if repo != b'.' else ''
for cmd in cleanup_cmds:
if show_debuginfo:
print("[DEBUG] Running{}: {}".format(location_info, ' '.join(cmd)))
ret = subproc.call(cmd, cwd=repo)
if ret != 0:
raise SystemExit("fatal: running '%s' failed!" % ' '.join(cmd))
if cmd[0:3] == 'git reflog expire'.split():
 self._write_stash()

def _get_rename(self, old_hash):

# If we already know the rename, just return it
new_hash = self._commit_renames.get(old_hash, None)
if new_hash:
return new_hash

# If it's not in the remaining pending renames, we don't know it

if old_hash is not None and old_hash not in self._pending_renames:
return None

# Read through the pending renames until we find it or we've read them all,
# and return whatever we might find
self._flush_renames(old_hash)
return self._commit_renames.get(old_hash, None)

def _flush_renames(self, old_hash=None, limit=0):

# Parse through self._pending_renames until we have read enough. We have
# read enough if:
# self._pending_renames is empty
# old_hash != None and we found a rename for old_hash
# limit > 0 and len(self._pending_renames) started less than 2*limit
# limit > 0 and len(self._pending_renames) < limit
if limit and len(self._pending_renames) < 2 * limit:
return
fi_input, fi_output = self._import_pipes
while self._pending_renames:
orig_hash, new_fast_export_id = self._pending_renames.popitem(last=False)
new_hash = fi_output.readline().rstrip()
self._commit_renames[orig_hash] = new_hash
self._graph.record_hash(new_fast_export_id, new_hash)
if old_hash == orig_hash:
return
if limit and len(self._pending_renames) < limit:
return

def _translate_commit_hash(self, matchobj_or_oldhash):

old_hash = matchobj_or_oldhash
if not isinstance(matchobj_or_oldhash, bytes):
old_hash = matchobj_or_oldhash.group(1)
orig_len = len(old_hash)
new_hash = self._get_rename(old_hash)
if new_hash is None:
if old_hash[0:7] not in self._commit_short_old_hashes:
self._commits_referenced_but_removed.add(old_hash)
return old_hash
possibilities = self._commit_short_old_hashes[old_hash[0:7]]
matches = [x for x in possibilities
if x[0:orig_len] == old_hash]
if len(matches) != 1:
self._commits_referenced_but_removed.add(old_hash)
return old_hash
old_hash = matches[0]
new_hash = self._get_rename(old_hash)

assert new_hash is not None

return new_hash[0:orig_len]
def _translate_full_commit_hash(self, matchobj):
old_hash = matchobj.group(1)
new_hash = self._get_rename(old_hash)
if new_hash is None:
return old_hash
return new_hash

def _maybe_trim_extra_parents(self, orig_parents, parents):

'''Due to pruning of empty commits, some parents could be non-existent
(None) or otherwise redundant. Remove the non-existent parents, and
remove redundant parents ***SO LONG AS*** that doesn't transform a
merge commit into a non-merge commit.

Returns a tuple:
(parents, new_first_parent_if_would_become_non_merge)'''

always_prune = (self._args.prune_degenerate == 'always')

# Pruning of empty commits means multiple things:

# * An original parent of this commit may have been pruned causing the
# need to rewrite the reported parent to the nearest ancestor. We
# want to know when we're dealing with such a parent.
# * Further, there may be no "nearest ancestor" if the entire history
# of that parent was also pruned. (Detectable by the parent being
# 'None')
# Remove all parents rewritten to None, and keep track of which parents
# were rewritten to an ancestor.
tmp = zip(parents,
orig_parents,
[(x in _SKIPPED_COMMITS or always_prune) for x in orig_parents])
tmp2 = [x for x in tmp if x[0] is not None]
if not tmp2:
# All ancestors have been pruned; we have no parents.
return [], None
parents, orig_parents, is_rewritten = [list(x) for x in zip(*tmp2)]

# We can't have redundant parents if we don't have at least 2 parents

if len(parents) < 2:
return parents, None

# Don't remove redundant parents if user doesn't want us to

if self._args.prune_degenerate == 'never':
return parents, None

# Remove duplicate parents (if both sides of history have lots of commits
# which become empty due to pruning, the most recent ancestor on both
# sides may be the same commit), except only remove parents that have
# been rewritten due to previous empty pruning.
seen = set()
seen_add = seen.add
# Deleting duplicate rewritten parents means keeping parents if either
# they have not been seen or they are ones that have not been rewritten.
parents_copy = parents
uniq = [[p, orig_parents[i], is_rewritten[i]] for i, p in enumerate(parents)
if not (p in seen or seen_add(p)) or not is_rewritten[i]]
parents, orig_parents, is_rewritten = [list(x) for x in zip(*uniq)]
if len(parents) < 2:
return parents_copy, parents[0]
 # Flatten unnecessary merges. (If one side of history is entirely
# empty commits that were pruned, we may end up attempting to
# merge a commit with its ancestor. Remove parents that are an
# ancestor of another parent.)
num_parents = len(parents)
to_remove = []
for cur in range(num_parents):
if not is_rewritten[cur]:
continue
for other in range(num_parents):
if cur == other:
continue
if not self._graph.is_ancestor(parents[cur], parents[other]):
continue
# parents[cur] is an ancestor of parents[other], so parents[cur]
# seems redundant. However, if it was intentionally redundant
# (e.g. a no-ff merge) in the original, then we want to keep it.
if not always_prune and \
self._orig_graph.is_ancestor(orig_parents[cur],
orig_parents[other]):
continue
# Some folks want their history to have all first parents be merge
# commits (except for any root commits), and always do a merge --no-ff.
# For such folks, don't remove the first parent even if it's an
# ancestor of other commits.
if self._args.no_ff and cur == 0:
continue
# Okay so the cur-th parent is an ancestor of the other-th parent,
# and it wasn't that way in the original repository; mark the
# cur-th parent as removable.
to_remove.append(cur)
break # cur removed, so skip rest of others -- i.e. check cur+=1
for x in reversed(to_remove):
parents.pop(x)
if len(parents) < 2:
return parents_copy, parents[0]

return parents, None

def _prunable(self, commit, new_1st_parent, had_file_changes, orig_parents):

parents = commit.parents

if self._args.prune_empty == 'never':
return False
always_prune = (self._args.prune_empty == 'always')

# For merge commits, unless there are prunable (redundant) parents, we

# do not want to prune
if len(parents) >= 2 and not new_1st_parent:
return False

if len(parents) < 2:
# Special logic for commits that started empty...
if not had_file_changes and not always_prune:
had_parents_pruned = (len(parents) < len(orig_parents) or
(len(orig_parents) == 1 and
orig_parents[0] in _SKIPPED_COMMITS))
# If the commit remains empty and had parents which were pruned,
# then prune this commit; otherwise, retain it
 return (not commit.file_changes and had_parents_pruned)

# We can only get here if the commit didn't start empty, so if it's
# empty now, it obviously became empty
if not commit.file_changes:
return True

# If there are no parents of this commit and we didn't match the case
# above, then this commit cannot be pruned. Since we have no parent(s)
# to compare to, abort now to prevent future checks from failing.
if not parents:
return False

# Similarly, we cannot handle the hard cases if we don't have a pipe

# to communicate with fast-import
if not self._import_pipes:
return False

# If there have not been renames/remappings of IDs (due to insertion of

# new blobs), then we can sometimes know things aren't prunable with a
# simple check
if not _IDS.has_renames():
# non-merge commits can only be empty if blob/file-change editing caused
# all file changes in the commit to have the same file contents as
# the parent.
changed_files = set(change.filename for change in commit.file_changes)
if len(orig_parents) < 2 and changed_files - self._files_tweaked:
return False

# Finally, the hard case: due to either blob rewriting, or due to pruning
# of empty commits wiping out the first parent history back to the merge
# base, the list of file_changes we have may not actually differ from our
# (new) first parent's version of the files, i.e. this would actually be
# an empty commit. Check by comparing the contents of this commit to its
# (remaining) parent.
#
# NOTE on why this works, for the case of original first parent history
# having been pruned away due to being empty:
# The first parent history having been pruned away due to being
# empty implies the original first parent would have a tree (after
# filtering) that matched the merge base's tree. Since
# file_changes has the changes needed to go from what would have
# been the first parent to our new commit, and what would have been
# our first parent has a tree that matches the merge base, then if
# the new first parent has a tree matching the versions of files in
# file_changes, then this new commit is empty and thus prunable.
fi_input, fi_output = self._import_pipes
self._flush_renames() # Avoid fi_output having other stuff present
# Optimization note: we could have two loops over file_changes, the
# first doing all the self._output.write() calls, and the second doing
# the rest. But I'm worried about fast-import blocking on fi_output
# buffers filling up so I instead read from it as I go.
for change in commit.file_changes:
parent = new_1st_parent or commit.parents[0] # exists due to above checks
quoted_filename = PathQuoting.enquote(change.filename)
if isinstance(parent, int):
self._output.write(b"ls :%d %s\n" % (parent, quoted_filename))
else:
self._output.write(b"ls %s %s\n" % (parent, quoted_filename))
 self._output.flush()
parent_version = fi_output.readline().split()
if change.type == b'D':
if parent_version != [b'missing', quoted_filename]:
return False
else:
blob_sha = change.blob_id
if isinstance(change.blob_id, int):
self._output.write(b"get-mark :%d\n" % change.blob_id)
self._output.flush()
blob_sha = fi_output.readline().rstrip()
if parent_version != [change.mode, b'blob', blob_sha, quoted_filename]:
return False

return True

def _record_remapping(self, commit, orig_parents):

new_id = None
# Record the mapping of old commit hash to new one
if commit.original_id and self._import_pipes:
fi_input, fi_output = self._import_pipes
self._output.write(b"get-mark :%d\n" % commit.id)
self._output.flush()
orig_id = commit.original_id
self._commit_short_old_hashes[orig_id[0:7]].add(orig_id)
# Note that we have queued up an id for later reading; flush a
# few of the older ones if we have too many queued up
self._pending_renames[orig_id] = commit.id
self._flush_renames(None, limit=40)
# Also, record if this was a merge commit that turned into a non-merge
# commit.
if len(orig_parents) >= 2 and len(commit.parents) < 2:
self._commits_no_longer_merges.append((commit.original_id, new_id))

def callback_metadata(self, extra_items = dict()):

return {'commit_rename_func': self._translate_commit_hash,
'ancestry_graph': self._graph,
'original_ancestry_graph': self._orig_graph,
**extra_items}

def _tweak_blob(self, blob):

if self._args.max_blob_size and len(blob.data) > self._args.max_blob_size:
blob.skip()

if blob.original_id in self._args.strip_blobs_with_ids:
blob.skip()

if ( self._args.replace_text
# not (if blob contains zero byte in the first 8Kb, that is, if blob is
binary data)
and not b"\0" in blob.data[0:8192]
):
for literal, replacement in self._args.replace_text['literals']:
blob.data = blob.data.replace(literal, replacement)
for regex, replacement in self._args.replace_text['regexes']:
blob.data = regex.sub(replacement, blob.data)

if self._blob_callback:
self._blob_callback(blob, self.callback_metadata())
def _filter_files(self, commit):
def filename_matches(path_expression, pathname):
''' Returns whether path_expression matches pathname or a leading
directory thereof, allowing path_expression to not have a trailing
slash even if it is meant to match a leading directory. '''
if path_expression == b'':
return True
n = len(path_expression)
if (pathname.startswith(path_expression) and
(path_expression[n-1:n] == b'/' or
len(pathname) == n or
pathname[n:n+1] == b'/')):
return True
return False

def newname(path_changes, pathname, use_base_name, filtering_is_inclusive):

''' Applies filtering and rename changes from path_changes to pathname,
returning any of None (file isn't wanted), original filename (file
is wanted with original name), or new filename. '''
wanted = False
full_pathname = pathname
if use_base_name:
pathname = os.path.basename(pathname)
for (mod_type, match_type, path_exp) in path_changes:
if mod_type == 'filter' and not wanted:
assert match_type in ('match', 'glob', 'regex')
if match_type == 'match' and filename_matches(path_exp, pathname):
wanted = True
if match_type == 'glob' and fnmatch.fnmatch(pathname, path_exp):
wanted = True
if match_type == 'regex' and path_exp.search(pathname):
wanted = True
elif mod_type == 'rename':
match, repl = path_exp
assert match_type in ('match','regex') # glob was translated to regex
if match_type == 'match' and filename_matches(match, full_pathname):
full_pathname = full_pathname.replace(match, repl, 1)
if match_type == 'regex':
full_pathname = match.sub(repl, full_pathname)
return full_pathname if (wanted == filtering_is_inclusive) else None

args = self._args
new_file_changes = {} # Assumes no renames or copies, otherwise collisions
for change in commit.file_changes:
# NEEDSWORK: _If_ we ever want to pass `--full-tree` to fast-export and
# parse that output, we'll need to modify this block; `--full-tree`
# issues a deleteall directive which has no filename, and thus this
# block would normally strip it. Of course, FileChange() and
# _parse_optional_filechange() would need updates too.
if change.type == b'DELETEALL':
new_file_changes[b''] = change
continue
if change.filename in self._newnames:
change.filename = self._newnames[change.filename]
else:
original_filename = change.filename
change.filename = newname(args.path_changes, change.filename,
args.use_base_name, args.inclusive)
 if self._filename_callback:
change.filename = self._filename_callback(change.filename)
self._newnames[original_filename] = change.filename
if not change.filename:
continue # Filtering criteria excluded this file; move on to next one
if change.filename in new_file_changes:
# Getting here means that path renaming is in effect, and caused one
# path to collide with another. That's usually bad, but can be okay
# under two circumstances:
# 1) Sometimes people have a file named OLDFILE in old revisions of
# history, and they rename to NEWFILE, and would like to rewrite
# history so that all revisions refer to it as NEWFILE. As such,
# we can allow a collision when (at least) one of the two paths
# is a deletion. Note that if OLDFILE and NEWFILE are unrelated
# this also allows the rewrite to continue, which makes sense
# since OLDFILE is no longer in the way.
# 2) If OLDFILE and NEWFILE are exactly equal, then writing them
# both to the same location poses no problem; we only need one
# file. (This could come up if someone copied a file in some
# commit, then later either deleted the file or kept it exactly
# in sync with the original with any changes, and then decides
# they want to rewrite history to only have one of the two files)
colliding_change = new_file_changes[change.filename]
if change.type == b'D':
# We can just throw this one away and keep the other
continue
elif change.type == b'M' and (
change.mode == colliding_change.mode and
change.blob_id == colliding_change.blob_id):
# The two are identical, so we can throw this one away and keep other
continue
elif new_file_changes[change.filename].type != b'D':
raise SystemExit(_("File renaming caused colliding pathnames!\n") +
_(" Commit: {}\n").format(commit.original_id) +
_(" Filename: {}").format(change.filename))
# Strip files that are too large
if self._args.max_blob_size and \
self._unpacked_size.get(change.blob_id, 0) > self._args.max_blob_size:
continue
if self._args.strip_blobs_with_ids and \
change.blob_id in self._args.strip_blobs_with_ids:
continue
# Otherwise, record the change
new_file_changes[change.filename] = change
commit.file_changes = [v for k,v in sorted(new_file_changes.items())]

def _tweak_commit(self, commit, aux_info):

if self._args.replace_message:
for literal, replacement in self._args.replace_message['literals']:
commit.message = commit.message.replace(literal, replacement)
for regex, replacement in self._args.replace_message['regexes']:
commit.message = regex.sub(replacement, commit.message)
if self._message_callback:
commit.message = self._message_callback(commit.message)

# Change the commit message according to callback

if not self._args.preserve_commit_hashes:
commit.message = self._hash_re.sub(self._translate_commit_hash,
commit.message)
 # Change the author & committer according to mailmap rules
args = self._args
if args.mailmap:
commit.author_name, commit.author_email = \
args.mailmap.translate(commit.author_name, commit.author_email)
commit.committer_name, commit.committer_email = \
args.mailmap.translate(commit.committer_name, commit.committer_email)
# Change author & committer according to callbacks
if self._name_callback:
commit.author_name = self._name_callback(commit.author_name)
commit.committer_name = self._name_callback(commit.committer_name)
if self._email_callback:
commit.author_email = self._email_callback(commit.author_email)
commit.committer_email = self._email_callback(commit.committer_email)

# Sometimes the 'branch' given is a tag; if so, rename it as requested so

# we don't get any old tagnames
if self._args.tag_rename:
commit.branch = RepoFilter._do_tag_rename(args.tag_rename, commit.branch)
if self._refname_callback:
commit.branch = self._refname_callback(commit.branch)

# Filter or rename the list of file changes

orig_file_changes = set(commit.file_changes)
self._filter_files(commit)

# Record ancestry graph

parents, orig_parents = commit.parents, aux_info['orig_parents']
if self._args.state_branch:
external_parents = parents
else:
external_parents = [p for p in parents if not isinstance(p, int)]
# The use of 'reversed' is intentional here; there is a risk that we have
# duplicates in parents, and we want to map from parents to the first
# entry we find in orig_parents in such cases.
parent_reverse_dict = dict(zip(reversed(parents), reversed(orig_parents)))

self._graph.record_external_commits(external_parents)
self._orig_graph.record_external_commits(external_parents)
self._graph.add_commit_and_parents(commit.id, parents) # new githash unknown
self._orig_graph.add_commit_and_parents(commit.old_id, orig_parents,
commit.original_id)

# Prune parents (due to pruning of empty commits) if relevant, note that

# new_1st_parent is None unless this was a merge commit that is becoming
# a non-merge
prev_1st_parent = parents[0] if parents else None
parents, new_1st_parent = self._maybe_trim_extra_parents(orig_parents,
parents)
commit.parents = parents

# If parents were pruned, then we need our file changes to be relative

# to the new first parent
#
# Notes:
# * new_1st_parent and new_1st_parent != parents[0] uniquely happens for
example when:
# working on merge, selecting subset of files and merge base still
# valid while first parent history doesn't touch any of those paths,
# but second parent history does. prev_1st_parent had already been
# rewritten to the non-None first ancestor and it remains valid.
# self._maybe_trim_extra_parents() avoids removing this first parent
# because it'd make the commit a non-merge. However, if there are
# no file_changes of note, we'll drop this commit and mark
# new_1st_parent as the new replacement. To correctly determine if
# there are no file_changes of note, we need to have the list of
# file_changes relative to new_1st_parent.
# (See t9390#3, "basic -> basic-ten using '--path ten'")
# * prev_1st_parent != parents[0] happens for example when:
# similar to above, but the merge base is no longer valid and was
# pruned away as well. Then parents started as e.g. [None, $num],
# and both prev_1st_parent and new_1st_parent are None, while parents
# after self._maybe_trim_extra_parents() becomes just [$num].
# (See t9390#67, "degenerate merge with non-matching filename".)
# Since $num was originally a second parent, we need to rewrite
# file changes to be relative to parents[0].
# * TODO: We should be getting the changes relative to the new first
# parent even if self._fep is None, BUT we can't. Our method of
# getting the changes right now is an external git diff invocation,
# which we can't do if we just have a fast export stream. We can't
# really work around it by querying the fast-import stream either,
# because the 'ls' directive only allows us to list info about
# specific paths, but we need to find out which paths exist in two
# commits and then query them. We could maybe force checkpointing in
# fast-import, then doing a diff from what'll be the new first parent
# back to prev_1st_parent (which may be None, i.e. empty tree), using
# the fact that in A->{B,C}->D, where D is merge of B & C, the diff
# from C->D == C->A + A->B + B->D, and in these cases A==B, so it
# simplifies to C->D == C->A + B->D, and C is our new 1st parent
# commit, A is prev_1st_commit, and B->D is commit.file_changes that
# we already have. However, checkpointing the fast-import process
# and figuring out how long to wait before we can run our diff just
# seems excessive. For now, just punt and assume the merge wasn't
# "evil" (i.e. that it's remerge-diff is empty, as is true for most
# merges). If the merge isn't evil, no further steps are necessary.
if parents and self._fep and (
prev_1st_parent != parents[0] or
new_1st_parent and new_1st_parent != parents[0]):
# Get the id from the original fast export stream corresponding to the
# new 1st parent. As noted above, that new 1st parent might be
# new_1st_parent, or if that is None, it'll be parents[0].
will_be_1st = new_1st_parent or parents[0]
old_id = parent_reverse_dict[will_be_1st]
# Now, translate that to a hash
will_be_1st_commit_hash = self._orig_graph.map_to_hash(old_id)
# Get the changes from what is going to be the new 1st parent to this
# merge commit. Note that since we are going from the new 1st parent
# to the merge commit, we can just replace the existing
# commit.file_changes rather than getting something we need to combine
# with the existing commit.file_changes. Also, we can just replace
# because prev_1st_parent is an ancestor of will_be_1st_commit_hash
# (or prev_1st_parent is None and first parent history is gone), so
# even if we retain prev_1st_parent and do not prune it, the changes
# will still work given the snapshot-based way fast-export/fast-import
# work.
commit.file_changes = GitUtils.get_file_changes(self._repo_working_dir,
will_be_1st_commit_hash,
 commit.original_id)

# Save these and filter them

orig_file_changes = set(commit.file_changes)
self._filter_files(commit)

# Find out which files were modified by the callbacks. Such paths could
# lead to subsequent commits being empty (e.g. if removing a line containing
# a password from every version of a file that had the password, and some
# later commit did nothing more than remove that line)
final_file_changes = set(commit.file_changes)
if self._args.replace_text or self._blob_callback:
differences = orig_file_changes.union(final_file_changes)
else:
differences = orig_file_changes.symmetric_difference(final_file_changes)
self._files_tweaked.update(x.filename for x in differences)

# Call the user-defined callback, if any

if self._commit_callback:
self._commit_callback(commit, self.callback_metadata(aux_info))

# Now print the resulting commit, or if prunable skip it

if not commit.dumped:
if not self._prunable(commit, new_1st_parent,
aux_info['had_file_changes'], orig_parents):
self._insert_into_stream(commit)
self._record_remapping(commit, orig_parents)
else:
rewrite_to = new_1st_parent or commit.first_parent()
commit.skip(new_id = rewrite_to)
if self._args.state_branch:
alias = Alias(commit.old_id or commit.id, rewrite_to or deleted_hash)
self._insert_into_stream(alias)
reset = Reset(commit.branch, rewrite_to or deleted_hash)
self._insert_into_stream(reset)
self._commit_renames[commit.original_id] = None

# Show progress
self._num_commits += 1
if not self._args.quiet:
self._progress_writer.show(self._parsed_message % self._num_commits)

@staticmethod
def _do_tag_rename(rename_pair, tagname):
old, new = rename_pair.split(b':', 1)
old, new = b'refs/tags/'+old, b'refs/tags/'+new
if tagname.startswith(old):
return tagname.replace(old, new, 1)
return tagname

def _tweak_tag(self, tag):

# Tweak the tag message according to callbacks
if self._args.replace_message:
for literal, replacement in self._args.replace_message['literals']:
tag.message = tag.message.replace(literal, replacement)
for regex, replacement in self._args.replace_message['regexes']:
tag.message = regex.sub(replacement, tag.message)
if self._message_callback:
tag.message = self._message_callback(tag.message)
 # Tweak the tag name according to tag-name-related callbacks
tag_prefix = b'refs/tags/'
fullref = tag_prefix+tag.ref
if self._args.tag_rename:
fullref = RepoFilter._do_tag_rename(self._args.tag_rename, fullref)
if self._refname_callback:
fullref = self._refname_callback(fullref)
if not fullref.startswith(tag_prefix):
msg = "Error: fast-import requires tags to be in refs/tags/ namespace."
msg += "\n {} renamed to {}".format(tag_prefix+tag.ref, fullref)
raise SystemExit(msg)
tag.ref = fullref[len(tag_prefix):]

# Tweak the tagger according to callbacks

if self._args.mailmap:
tag.tagger_name, tag.tagger_email = \
self._args.mailmap.translate(tag.tagger_name, tag.tagger_email)
if self._name_callback:
tag.tagger_name = self._name_callback(tag.tagger_name)
if self._email_callback:
tag.tagger_email = self._email_callback(tag.tagger_email)

# Call general purpose tag callback

if self._tag_callback:
self._tag_callback(tag, self.callback_metadata())

def _tweak_reset(self, reset):

if self._args.tag_rename:
reset.ref = RepoFilter._do_tag_rename(self._args.tag_rename, reset.ref)
if self._refname_callback:
reset.ref = self._refname_callback(reset.ref)
if self._reset_callback:
self._reset_callback(reset, self.callback_metadata())

def results_tmp_dir(self, create_if_missing=True):

target_working_dir = self._args.target or b'.'
git_dir = GitUtils.determine_git_dir(target_working_dir)
d = os.path.join(git_dir, b'filter-repo')
if create_if_missing and not os.path.isdir(d):
os.mkdir(d)
return d

def _load_marks_file(self, marks_basename):

full_branch = 'refs/heads/{}'.format(self._args.state_branch)
marks_file = os.path.join(self.results_tmp_dir(), marks_basename)
working_dir = self._args.target or b'.'
cmd = ['git', '-C', working_dir, 'show-ref', full_branch]
contents = b''
if subproc.call(cmd, stdout=subprocess.DEVNULL) == 0:
cmd = ['git', '-C', working_dir, 'show',
'%s:%s' % (full_branch, decode(marks_basename))]
try:
contents = subproc.check_output(cmd)
except subprocess.CalledProcessError as e: # pragma: no cover
raise SystemExit(_("Failed loading %s from %s") %
(decode(marks_basename), full_branch))
if contents:
biggest_id = max(int(x.split()[0][1:]) for x in contents.splitlines())
 _IDS._next_id = max(_IDS._next_id, biggest_id+1)
with open(marks_file, 'bw') as f:
f.write(contents)
return marks_file

def _save_marks_files(self):
basenames = [b'source-marks', b'target-marks']
working_dir = self._args.target or b'.'

# Check whether the branch exists

parent = []
full_branch = 'refs/heads/{}'.format(self._args.state_branch)
cmd = ['git', '-C', working_dir, 'show-ref', full_branch]
if subproc.call(cmd, stdout=subprocess.DEVNULL) == 0:
parent = ['-p', full_branch]

# Run 'git hash-object $MARKS_FILE' for each marks file, save result
blob_hashes = {}
for marks_basename in basenames:
marks_file = os.path.join(self.results_tmp_dir(), marks_basename)
if not os.path.isfile(marks_file): # pragma: no cover
raise SystemExit(_("Failed to find %s to save to %s")
% (marks_file, self._args.state_branch))
cmd = ['git', '-C', working_dir, 'hash-object', '-w', marks_file]
blob_hashes[marks_basename] = subproc.check_output(cmd).strip()

# Run 'git mktree' to create a tree out of it

p = subproc.Popen(['git', '-C', working_dir, 'mktree'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
for b in basenames:
p.stdin.write(b'100644 blob %s\t%s\n' % (blob_hashes[b], b))
p.stdin.close()
p.wait()
tree = p.stdout.read().strip()

# Create the new commit

cmd = (['git', '-C', working_dir, 'commit-tree', '-m', 'New mark files',
tree] + parent)
commit = subproc.check_output(cmd).strip()
subproc.call(['git', '-C', working_dir, 'update-ref', full_branch, commit])

def importer_only(self):
self._run_sanity_checks()
self._setup_output()

def set_output(self, outputRepoFilter):

assert outputRepoFilter._output

# set_output implies this RepoFilter is doing exporting, though may not

# be the only one.
self._setup_input(use_done_feature = False)

# Set our output management up to pipe to outputRepoFilter's locations

self._managed_output = False
self._output = outputRepoFilter._output
self._import_pipes = outputRepoFilter._import_pipes

# Handle sanity checks, though currently none needed for export-only cases
self._run_sanity_checks()
def _read_stash(self):
if self._orig_refs and b'refs/stash' in self._orig_refs and \
self._args.refs == ['--all']:
repo_working_dir = self._args.source or b'.'
git_dir = GitUtils.determine_git_dir(repo_working_dir)
stash = os.path.join(git_dir, b'logs', b'refs', b'stash')
if os.path.exists(stash):
with open(stash, 'br') as f:
self._stash = f.read()
out = subproc.check_output('git rev-list -g refs/stash'.split(),
cwd=repo_working_dir)
self._args.refs.extend(decode(out.strip()).split())

def _write_stash(self):
if self._stash:
target_working_dir = self._args.target or b'.'
git_dir = GitUtils.determine_git_dir(target_working_dir)
stash = os.path.join(git_dir, b'logs', b'refs', b'stash')
with open(stash, 'bw') as f:
self._stash = self._full_hash_re.sub(self._translate_full_commit_hash,
self._stash)
f.write(self._stash)
print(_("Rewrote the stash."))

def _setup_input(self, use_done_feature):

if self._args.stdin:
self._input = sys.stdin.detach()
sys.stdin = None # Make sure no one tries to accidentally use it
self._fe_orig = None
else:
self._read_stash()
skip_blobs = (self._blob_callback is None and
self._args.replace_text is None and
self._args.source == self._args.target)
extra_flags = []
if skip_blobs:
extra_flags.append('--no-data')
if self._args.max_blob_size:
self._unpacked_size, packed_size = GitUtils.get_blob_sizes()
if use_done_feature:
extra_flags.append('--use-done-feature')
if write_marks:
extra_flags.append(b'--mark-tags')
if self._args.state_branch:
assert(write_marks)
source_marks_file = self._load_marks_file(b'source-marks')
extra_flags.extend([b'--export-marks='+source_marks_file,
b'--import-marks='+source_marks_file])
if self._args.preserve_commit_encoding is not None: # pragma: no cover
reencode = 'no' if self._args.preserve_commit_encoding else 'yes'
extra_flags.append('--reencode='+reencode)
if self._args.date_order:
extra_flags.append('--date-order')
location = ['-C', self._args.source] if self._args.source else []
fep_cmd = ['git'] + location + ['fast-export', '--show-original-ids',
'--signed-tags=strip', '--tag-of-filtered-object=rewrite',
'--fake-missing-tagger', '--reference-excluded-parents'
] + extra_flags + self._args.refs
 self._fep = subproc.Popen(fep_cmd, bufsize=-1, stdout=subprocess.PIPE)
self._input = self._fep.stdout
if self._args.dry_run or self._args.debug:
self._fe_orig = os.path.join(self.results_tmp_dir(),
b'fast-export.original')
output = open(self._fe_orig, 'bw')
self._input = InputFileBackup(self._input, output)
if self._args.debug:
tmp = [decode(x) if isinstance(x, bytes) else x for x in fep_cmd]
print("[DEBUG] Running: {}".format(' '.join(tmp)))
print(" (saving a copy of the output at {})"
.format(decode(self._fe_orig)))

def _setup_output(self):
if not self._args.dry_run:
location = ['-C', self._args.target] if self._args.target else []
fip_cmd = ['git'] + location + ['-c', 'core.ignorecase=false',
'fast-import', '--force', '--quiet']
if date_format_permissive:
fip_cmd.append('--date-format=raw-permissive')
if self._args.state_branch:
target_marks_file = self._load_marks_file(b'target-marks')
fip_cmd.extend([b'--export-marks='+target_marks_file,
b'--import-marks='+target_marks_file])
self._fip = subproc.Popen(fip_cmd, bufsize=-1,
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
self._import_pipes = (self._fip.stdin, self._fip.stdout)
if self._args.dry_run or self._args.debug:
self._fe_filt = os.path.join(self.results_tmp_dir(),
b'fast-export.filtered')
self._output = open(self._fe_filt, 'bw')
else:
self._output = self._fip.stdin
if self._args.debug and not self._args.dry_run:
self._output = DualFileWriter(self._fip.stdin, self._output)
tmp = [decode(x) if isinstance(x, bytes) else x for x in fip_cmd]
print("[DEBUG] Running: {}".format(' '.join(tmp)))
print(" (using the following file as input: {})"
.format(decode(self._fe_filt)))

def _migrate_origin_to_heads(self):
refs_to_migrate = set(x for x in self._orig_refs
if x.startswith(b'refs/remotes/origin/'))
if not refs_to_migrate:
return
if self._args.debug:
print("[DEBUG] Migrating refs/remotes/origin/* -> refs/heads/*")
target_working_dir = self._args.target or b'.'
p = subproc.Popen('git update-ref --no-deref --stdin'.split(),
stdin=subprocess.PIPE, cwd=target_working_dir)
for ref in refs_to_migrate:
if ref == b'refs/remotes/origin/HEAD':
p.stdin.write(b'delete %s %s\n' % (ref, self._orig_refs[ref]))
del self._orig_refs[ref]
continue
newref = ref.replace(b'refs/remotes/origin/', b'refs/heads/')
if newref not in self._orig_refs:
p.stdin.write(b'create %s %s\n' % (newref, self._orig_refs[ref]))
p.stdin.write(b'delete %s %s\n' % (ref, self._orig_refs[ref]))
 self._orig_refs[newref] = self._orig_refs[ref]
del self._orig_refs[ref]
p.stdin.close()
if p.wait():
raise SystemExit(_("git update-ref failed; see above")) # pragma: no cover

# Now remove the origin remote

if b'remote.origin.url' not in self._config_settings:
return
url = self._config_settings[b'remote.origin.url'].decode(errors='replace')
m = _("NOTICE: Removing 'origin' remote; see 'Why is my origin removed?'\n"
" in the manual if you want to push back there.\n"
" (was %s)") % url
print(m)
subproc.call('git remote rm origin'.split(), cwd=target_working_dir)

def _final_commands(self):
self._finalize_handled = True
self._done_callback and self._done_callback()

if not self._args.quiet:
self._progress_writer.finish()

def _ref_update(self, target_working_dir):

# Start the update-ref process
p = subproc.Popen('git update-ref --no-deref --stdin'.split(),
stdin=subprocess.PIPE,
cwd=target_working_dir)

# Remove replace_refs from _orig_refs

replace_refs = {k:v for k, v in self._orig_refs.items()
if k.startswith(b'refs/replace/')}
reverse_replace_refs = collections.defaultdict(list)
for k,v in replace_refs.items():
reverse_replace_refs[v].append(k)
all(map(self._orig_refs.pop, replace_refs))

# Remove unused refs

exported_refs, imported_refs = self.get_exported_and_imported_refs()
refs_to_nuke = exported_refs - imported_refs
if self._args.partial:
refs_to_nuke = set()
if refs_to_nuke and self._args.debug:
print("[DEBUG] Deleting the following refs:\n "+
decode(b"\n ".join(sorted(refs_to_nuke))))
p.stdin.write(b''.join([b"delete %s\n" % x
for x in refs_to_nuke]))

# Delete or update and add replace_refs; note that fast-export automatically

# handles 'update-no-add', we only need to take action for the other four
# choices for replace_refs.
self._flush_renames()
actual_renames = {k:v for k,v in self._commit_renames.items() if k != v}
if self._args.replace_refs in ['delete-no-add', 'delete-and-add']:
# Delete old replace refs, if unwanted
replace_refs_to_nuke = set(replace_refs)
if self._args.replace_refs == 'delete-and-add':
# git-update-ref won't allow us to update a ref twice, so be careful
# to avoid deleting refs we'll later update
 replace_refs_to_nuke = replace_refs_to_nuke.difference(
[b'refs/replace/'+x for x in actual_renames])
p.stdin.write(b''.join([b"delete %s\n" % x
for x in replace_refs_to_nuke]))
if self._args.replace_refs in ['delete-and-add', 'update-or-add',
'update-and-add']:
# Add new replace refs
update_only = (self._args.replace_refs == 'update-or-add')
p.stdin.write(b''.join([b"update refs/replace/%s %s\n" % (old, new)
for old,new in actual_renames.items()
if new and not (update_only and
old in reverse_replace_refs)]))

# Complete the update-ref process

p.stdin.close()
if p.wait():
raise SystemExit(_("git update-ref failed; see above")) # pragma: no cover

def _remap_to(self, oldish_hash):

'''
Given an oldish_hash (from the beginning of the current run), return:
IF oldish_hash is NOT pruned:
the hash of the rewrite of oldish_hash
otherwise:
the hash of the rewrite of the first unpruned ancestor of oldish_hash
'''
old_id = self._orig_graph._hash_to_id[oldish_hash]
new_id = _IDS.translate(old_id)
new_hash = self._graph.git_hash[new_id] if new_id else deleted_hash
return new_hash

def _compute_metadata(self, metadata_dir, orig_refs):

already_ran = os.path.isfile(os.path.join(metadata_dir, b'already_ran'))

#
# First, handle commit_renames
#
old_commit_renames = dict()
if not already_ran:
commit_renames = {old: new
for old, new in self._commit_renames.items()
}
else:
# Read commit-map into old_commit_renames
with open(os.path.join(metadata_dir, b'commit-map'), 'br') as f:
f.readline() # Skip the header line
for line in f:
(old,new) = line.split()
old_commit_renames[old] = new
# Use A->B mappings in old_commit_renames, and B->C mappings in
# self._commit_renames to yield A->C mappings in commit_renames
commit_renames = {old: self._commit_renames.get(newish, newish)
for old, newish in old_commit_renames.items()}
# If there are any B->C mappings in self._commit_renames for which
# there was no A->B mapping in old_commit_renames, then add the
# B->C mapping to commit_renames too.
seen = set(old_commit_renames.values())
commit_renames.update({old: new
for old, new in self._commit_renames.items()
 if old not in seen})

#
# Second, handle ref_maps
#
exported_refs, imported_refs = self.get_exported_and_imported_refs()

old_commit_unrenames = dict()
if not already_ran:
old_ref_map = dict((refname, (old_hash, deleted_hash))
for refname, old_hash in orig_refs.items()
if refname in exported_refs)
else:
# old_commit_renames talk about how commits were renamed in the original
# run. Let's reverse it to find out how to get from the intermediate
# commit name, back to the original. Because everything in orig_refs
# right now refers to the intermediate commits after the first run(s),
# and we need to map them back to what they were before any changes.
old_commit_unrenames = dict((v,k) for (k,v) in old_commit_renames.items())

old_ref_map = {}
# Populate old_ref_map from the 'ref-map' file
with open(os.path.join(metadata_dir, b'ref-map'), 'br') as f:
f.readline() # Skip the header line
for line in f:
(old,intermediate,ref) = line.split()
old_ref_map[ref] = (old, intermediate)
# Append to old_ref_map items from orig_refs that were exported, but
# get the actual original commit name
for refname, old_hash in orig_refs.items():
if refname in old_ref_map:
continue
if refname not in exported_refs:
continue
# Compute older_hash
original_hash = old_commit_unrenames.get(old_hash, old_hash)
old_ref_map[refname] = (original_hash, deleted_hash)

new_refs = {}
new_refs_initialized = False
ref_maps = {}
self._orig_graph._ensure_reverse_maps_populated()
for refname, pair in old_ref_map.items():
old_hash, hash_ref_becomes_if_not_imported_in_this_run = pair
if refname not in imported_refs:
new_hash = hash_ref_becomes_if_not_imported_in_this_run
elif old_hash in commit_renames:
intermediate = old_commit_renames.get(old_hash,old_hash)
if intermediate in self._commit_renames:
new_hash = self._remap_to(intermediate)
else:
new_hash = intermediate
else: # Must be either an annotated tag, or a ref whose tip was pruned
if not new_refs_initialized:
target_working_dir = self._args.target or b'.'
new_refs = GitUtils.get_refs(target_working_dir)
if refname in new_refs:
new_hash = new_refs[refname]
else:
 new_hash = deleted_hash
ref_maps[refname] = (old_hash, new_hash)
if self._args.source or self._args.target:
if not new_refs_initialized:
target_working_dir = self._args.target or b'.'
new_refs = GitUtils.get_refs(target_working_dir)
for ref, new_hash in new_refs.items():
if ref not in orig_refs and not ref.startswith(b'refs/replace/'):
old_hash = b'0'*len(new_hash)
ref_maps[ref] = (old_hash, new_hash)

#
# Third, handle first_changes
#

old_first_changes = dict()
if already_ran:
# Read first_changes into old_first_changes
with open(os.path.join(metadata_dir, b'first-changed-commits'), 'br') as f:
for line in f:
changed_commit, undeleted_self_or_ancestor = line.strip().split()
old_first_changes[changed_commit] = undeleted_self_or_ancestor
# We need to find the commits that were modified whose parents were not.
# To be able to find parents, we need the commit names as of the beginning
# of this run, and then when we are done, we need to map them back to the
# name of the commits from before any git-filter-repo runs.
#
# We are excluding here any commits deleted in previous git-filter-repo
# runs
undo_old_commit_renames = dict((v,k) for (k,v) in old_commit_renames.items()
if v != deleted_hash)
# Get a list of all commits that were changed, as of the beginning of
# this latest run.
changed_commits = {new
for (old,new) in old_commit_renames.items()
if old != new and new != deleted_hash} | \
{old
for (old,new) in self._commit_renames.items()
if old != new}
special_changed_commits = {old
for (old,new) in old_commit_renames.items()
if new == deleted_hash}
first_changes = dict()
for (old,new) in self._commit_renames.items():
if old == new:
# old wasn't modified, can't be first change if not even a change
continue
if old_commit_unrenames.get(old,old) != old:
# old was already modified in previous run; while it might represent
# something that is still a first change, we'll handle that as we
# loop over old_first_changes below
continue
if any(parent in changed_commits
for parent in self._orig_graph.get_parent_hashes(old)):
# a parent of old was modified, so old is not a first change
continue
# At this point, old IS a first change. We need to find out what new
# commit it maps to, or if it doesn't map to one, what new commit was
# its most recent ancestor that wasn't pruned.
 if new is None:
new = self._remap_to(old)
first_changes[old] = (new if new is not None else deleted_hash)
for (old,undeleted_self_or_ancestor) in old_first_changes.items():
if undeleted_self_or_ancestor == deleted_hash:
# old represents a commit that was pruned and whose entire ancestry
# was pruned. So, old is still a first change
first_changes[old] = undeleted_self_or_ancestor
continue
intermediate = old_commit_renames.get(old, old)
usoa = undeleted_self_or_ancestor
new_ancestor = self._commit_renames.get(usoa, usoa)
if intermediate == deleted_hash:
# old was pruned in previous rewrite
if usoa != new_ancestor:
# old's ancestor got rewritten in this filtering run; we can drop
# this one from first_changes.
continue
# Getting here means old was a first change and old was pruned in a
# previous run, and its ancestors that survived were non rewritten in
# this run, so old remains a first change
first_changes[old] = new_ancestor # or usoa, since new_ancestor == usoa
continue
assert(usoa == intermediate) # old wasn't pruned => usoa == intermediate

# Check whether parents of intermediate were rewritten. Note that

# intermediate in self._commit_renames only means that intermediate was
# processed by the latest filtering (not necessarily that it changed),
# but we need to know that before we can check for parent hashes having
# changed.
if intermediate not in self._commit_renames:
# This commit was not processed by this run, so it remains a first
# change
first_changes[old] = usoa
continue
if any(parent in changed_commits
for parent in self._orig_graph.get_parent_hashes(intermediate)):
# An ancestor was modified by this run, so it is no longer a first
# change; continue to the next one.
continue
# This change is a first_change; find the new commit its usoa maps to
new = self._remap_to(intermediate)
assert(new is not None)
first_changes[old] = new

return commit_renames, ref_maps, first_changes

def _record_metadata(self, metadata_dir, orig_refs):

self._flush_renames()
commit_renames, ref_maps, first_changes = \
self._compute_metadata(metadata_dir, orig_refs)

with open(os.path.join(metadata_dir, b'commit-map'), 'bw') as f:

f.write(("%-40s %s\n" % (_("old"), _("new"))).encode())
for (old,new) in sorted(commit_renames.items()):
msg = b'%s %s\n' % (old, new if new != None else deleted_hash)
f.write(msg)

with open(os.path.join(metadata_dir, b'ref-map'), 'bw') as f:

 f.write(("%-40s %-40s %s\n" % (_("old"), _("new"), _("ref"))).encode())
for refname, hash_pair in sorted(ref_maps.items()):
(old_hash, new_hash) = hash_pair
f.write(b'%s %s %s\n' % (old_hash, new_hash, refname))

with open(os.path.join(metadata_dir, b'first-changed-commits'), 'bw') as f:

for commit, undeleted_self_or_ancestor in sorted(first_changes.items()):
f.write(b'%s %s\n' % (commit, undeleted_self_or_ancestor))

with open(os.path.join(metadata_dir, b'suboptimal-issues'), 'bw') as f:

issues_found = False
if self._commits_no_longer_merges:
issues_found = True

f.write(textwrap.dedent(_('''
The following commits used to be merge commits but due to filtering
are now regular commits; they likely have suboptimal commit messages
(e.g. "Merge branch next into master"). Original commit hash on the
left, commit hash after filtering/rewriting on the right:
''')[1:]).encode())
for oldhash, newhash in self._commits_no_longer_merges:
f.write(' {} {}\n'.format(oldhash, newhash).encode())
f.write(b'\n')

if self._commits_referenced_but_removed:
issues_found = True
f.write(textwrap.dedent(_('''
The following commits were filtered out, but referenced in another
commit message. The reference to the now-nonexistent commit hash
(or a substring thereof) was left as-is in any commit messages:
''')[1:]).encode())
for bad_commit_reference in self._commits_referenced_but_removed:
f.write(' {}\n'.format(bad_commit_reference).encode())
f.write(b'\n')

if not issues_found:
f.write(_("No filtering problems encountered.\n").encode())

with open(os.path.join(metadata_dir, b'already_ran'), 'bw') as f:

f.write(_("This file exists to allow you to filter again without --force,\n"
"and to specify that metadata files should be updated instead\n"
"of rewritten").encode())

def finish(self):
''' Alternative to run() when there is no input of our own to parse,
meaning that run only really needs to close the handle to fast-import
and let it finish, thus making a call to "run" feel like a misnomer. '''
assert not self._input
assert self._managed_output
self.run()

def insert(self, obj, direct_insertion = False):

if not direct_insertion:
if type(obj) == Blob:
self._tweak_blob(obj)
elif type(obj) == Commit:
aux_info = {'orig_parents': obj.parents,
'had_file_changes': bool(obj.file_changes)}
self._tweak_commit(obj, aux_info)
 elif type(obj) == Reset:
self._tweak_reset(obj)
elif type(obj) == Tag:
self._tweak_tag(obj)
self._insert_into_stream(obj)

def _insert_into_stream(self, obj):

if not obj.dumped:
if self._parser:
self._parser.insert(obj)
else:
obj.dump(self._output)

def get_exported_and_imported_refs(self):
return self._parser.get_exported_and_imported_refs()

def run(self):
start = time.time()
if not self._input and not self._output:
self._run_sanity_checks()
if not self._args.dry_run and not self._args.partial:
self._migrate_origin_to_heads()
self._setup_input(use_done_feature = True)
self._setup_output()
assert self._sanity_checks_handled

if self._input:
# Create and run the filter
self._repo_working_dir = self._args.source or b'.'
self._parser = FastExportParser(blob_callback = self._tweak_blob,
commit_callback = self._tweak_commit,
tag_callback = self._tweak_tag,
reset_callback = self._tweak_reset,
done_callback = self._final_commands)
self._parser.run(self._input, self._output)
if not self._finalize_handled:
self._final_commands()

# Make sure fast-export completed successfully

if not self._args.stdin and self._fep.wait():
raise SystemExit(_("Error: fast-export failed; see above.")) # pragma: no
cover
self._input.close()

# If we're not the manager of self._output, we should avoid post-run cleanup

if not self._managed_output:
return

# Close the output and ensure fast-import successfully completes

self._output.close()
if not self._args.dry_run and self._fip.wait():
raise SystemExit(_("Error: fast-import failed; see above.")) # pragma: no
cover

# With fast-export and fast-import complete, update state if requested

if self._args.state_branch:
self._save_marks_files()

# Notify user how long it took, before doing a gc and such

 msg = "New history written in {:.2f} seconds..."
if self._args.repack:
msg = "New history written in {:.2f} seconds; now repacking/cleaning..."
print(msg.format(time.time()-start))

# Exit early, if requested

if self._args.dry_run:
print(_("NOTE: Not running fast-import or cleaning up; --dry-run passed."))
if self._fe_orig:
print(_(" Requested filtering can be seen by comparing:"))
print(" " + decode(self._fe_orig))
else:
print(_(" Requested filtering can be seen at:"))
print(" " + decode(self._fe_filt))
return

target_working_dir = self._args.target or b'.'

if self._input:
self._ref_update(target_working_dir)

# Write out data about run

self._record_metadata(self.results_tmp_dir(), self._orig_refs)

# Final cleanup:
# If we need a repack, then nuke the reflogs and repack.
# If we need a reset, do a reset --hard
reset = not GitUtils.is_repository_bare(target_working_dir)
self.cleanup(target_working_dir, self._args.repack, reset,
run_quietly=self._args.quiet,
show_debuginfo=self._args.debug)

# Let user know how long it took

print(_("Completely finished after {:.2f} seconds.")
.format(time.time()-start))

def main():
setup_gettext()
args = FilteringOptions.parse_args(sys.argv[1:])
if args.analyze:
RepoAnalyze.run(args)
else:
filter = RepoFilter(args)
filter.run()

if __name__ == '__main__':
main()