From 26d59e8442b74932488f5cf4167cfec4e7dc0c45 Mon Sep 17 00:00:00 2001 From: Tony Duckles Date: Sat, 11 Feb 2012 15:36:21 -0600 Subject: [PATCH] Add --keep-date, --keep-prop, --archive, --log-author, --log-date command line args Add hook-examples/pre-revprop-change_example.txt --- hook-examples/pre-revprop-change_example.txt | 70 +++++++++++ svn2svn/__init__.py | 2 +- svn2svn/run/parse.py | 3 + svn2svn/run/svn2svn.py | 116 ++++++++++++------- svn2svn/svnclient.py | 6 +- tests/make-replay-repo.sh | 7 +- 6 files changed, 154 insertions(+), 50 deletions(-) create mode 100644 hook-examples/pre-revprop-change_example.txt diff --git a/hook-examples/pre-revprop-change_example.txt b/hook-examples/pre-revprop-change_example.txt new file mode 100644 index 0000000..d5451d4 --- /dev/null +++ b/hook-examples/pre-revprop-change_example.txt @@ -0,0 +1,70 @@ +#!/bin/sh + +# PRE-REVPROP-CHANGE HOOK + +# The pre-revprop-change hook is invoked before a revision property +# is added, modified or deleted. Subversion runs this hook by invoking +# a program (script, executable, binary, etc.) named 'pre-revprop-change' +# (for which this file is a template), with the following ordered +# arguments: +# +# [1] REPOS-PATH (the path to this repository) +# [2] REVISION (the revision being tweaked) +# [3] USER (the username of the person tweaking the property) +# [4] PROPNAME (the property being set on the revision) +# [5] ACTION (the property is being 'A'dded, 'M'odified, or 'D'eleted) +# +# [STDIN] PROPVAL ** the new property value is passed via STDIN. +# +# If the hook program exits with success, the propchange happens; but +# if it exits with failure (non-zero), the propchange doesn't happen. +# The hook program can use the 'svnlook' utility to examine the +# existing value of the revision property. +# +# WARNING: unlike other hooks, this hook MUST exist for revision +# properties to be changed. If the hook does not exist, Subversion +# will behave as if the hook were present, but failed. The reason +# for this is that revision properties are UNVERSIONED, meaning that +# a successful propchange is destructive; the old value is gone +# forever. We recommend the hook back up the old value somewhere. +# +# On a Unix system, the normal procedure is to have 'pre-revprop-change' +# invoke other programs to do the real work, though it may do the +# work itself too. +# +# Note that 'pre-revprop-change' must be executable by the user(s) who will +# invoke it (typically the user httpd runs as), and that user must +# have filesystem-level permission to access the repository. +# +# On a Windows system, you should name the hook program +# 'pre-revprop-change.bat' or 'pre-revprop-change.exe', +# but the basic idea is the same. +# +# The hook program typically does not inherit the environment of +# its parent process. For example, a common problem is for the +# PATH environment variable to not be set to its usual value, so +# that subprograms fail to launch unless invoked via absolute path. +# If you're having unexpected problems with a hook program, the +# culprit may be unusual (or missing) environment variables. +# +# Here is an example hook script, for a Unix /bin/sh interpreter. +# For more examples and pre-written hooks, see those in +# the Subversion repository at +# http://svn.apache.org/repos/asf/subversion/trunk/tools/hook-scripts/ and +# http://svn.apache.org/repos/asf/subversion/trunk/contrib/hook-scripts/ + +REPOS="$1" +REV="$2" +USER="$3" +PROPNAME="$4" +ACTION="$5" + +echo "pre-revprop-change: REPOS=$1 REV=$2 USER=$3 PROPNAME=$4 ACTION=$5" >&2 + +# Allow modifying certain properties +if [ "$ACTION" = "M" -a "$PROPNAME" = "svn:date" ]; then exit 0; fi +if [ "$ACTION" = "M" -a "$PROPNAME" = "svn:author" ]; then exit 0; fi + +# Reject everything else +echo "Changing revision property $PROPNAME is prohibited" >&2 +exit 1 diff --git a/svn2svn/__init__.py b/svn2svn/__init__.py index 2d3f101..bdca953 100644 --- a/svn2svn/__init__.py +++ b/svn2svn/__init__.py @@ -2,7 +2,7 @@ __all__ = [] __author__ = 'Tony Duckles' __license__ = 'GNU General Public License (version 3 or later)' -__versioninfo__ = (1, 1, 0) +__versioninfo__ = (1, 2, 0) base_version = '.'.join(map(str, __versioninfo__)) full_version = base_version diff --git a/svn2svn/run/parse.py b/svn2svn/run/parse.py index 0dabdb7..8b378e4 100644 --- a/svn2svn/run/parse.py +++ b/svn2svn/run/parse.py @@ -61,3 +61,6 @@ class HelpFormatter(optparse.IndentedHelpFormatter): elif opts[-1] != "\n": result.append("\n") return "".join(result) + + def format_usage(self, usage): + return usage diff --git a/svn2svn/run/svn2svn.py b/svn2svn/run/svn2svn.py index 5e32716..83c33f5 100644 --- a/svn2svn/run/svn2svn.py +++ b/svn2svn/run/svn2svn.py @@ -45,10 +45,14 @@ def commit_from_svn_log_entry(log_entry, commit_paths=None, target_revprops=None # Uncomment this one one if you prefer UTC commit times #svn_date = "%d 0" % timestamp args = ["commit", "--force-log"] + message = log_entry['message'] + if options.log_date: + message += "\nDate: " + svn_date + if options.log_author: + message += "\nAuthor: " + log_entry['author'] if options.keep_author: - args += ["-m", log_entry['message'] + "\nDate: " + svn_date, "--username", log_entry['author']] - else: - args += ["-m", log_entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + log_entry['author']] + args += ["--username", log_entry['author']] + args += ["-m", message] revprops = {} if log_entry['revprops']: # Carry forward any revprop's from the source revision @@ -300,7 +304,7 @@ def build_rev_map(target_url, target_end_rev, source_info): """ global rev_map rev_map = {} - ui.status("Rebuilding rev_map...", level=ui.VERBOSE) + ui.status("Rebuilding target_rev -> source_rev rev_map...", level=ui.VERBOSE) proc_count = 0 it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True) for log_entry in it_log_entries: @@ -316,15 +320,15 @@ def build_rev_map(target_url, target_end_rev, source_info): target_rev = log_entry['revision'] set_rev_map(source_rev, target_rev) -def get_svn_dirlist(svn_path, svn_rev = ""): +def get_svn_dirlist(svn_path, rev_number = ""): """ Get a list of all the child contents (recusive) of the given folder path. """ args = ["list"] path = svn_path - if svn_rev: - args += ["-r", svn_rev] - path += "@"+str(svn_rev) + if rev_number: + args += ["-r", rev_number] + path += "@"+str(rev_number) args += [path] paths = run_svn(args, no_fail=True) paths = paths.strip("\n").split("\n") if len(paths)>1 else [] @@ -426,8 +430,8 @@ def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom # Export the final verison of this file. run_svn(["export", "--force", "-r", source_rev, source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset]) - # Copy SVN properties from source repo - sync_svn_props(source_url, source_rev, path_offset) + if options.keep_prop: + sync_svn_props(source_url, source_rev, path_offset) else: ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN') else: @@ -450,8 +454,8 @@ def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset]) # If not already under version-control, then "svn add" this file/folder. run_svn(["add", "--parents", path_offset]) - # Copy SVN properties from source repo - sync_svn_props(source_url, source_rev, path_offset) + if options.keep_prop: + sync_svn_props(source_url, source_rev, path_offset) if is_dir: # For any folders that we process, process any child contents, so that we correctly # replay copies/replaces/etc. @@ -584,8 +588,8 @@ def process_svn_log_entry(log_entry, commit_paths, prefix = ""): # Need to use in_svn here to handle cases where client committed the parent # folder and each indiv sub-folder. run_svn(["add", "--parents", path_offset]) - # Copy SVN properties from source repo - sync_svn_props(source_url, source_rev, path_offset) + if options.keep_prop: + sync_svn_props(source_url, source_rev, path_offset) elif action == 'D': run_svn(["remove", "--force", path_offset]) @@ -594,8 +598,8 @@ def process_svn_log_entry(log_entry, commit_paths, prefix = ""): if path_is_file: run_svn(["export", "--force", "-N" , "-r", source_rev, source_url+"/"+path_offset+"@"+str(source_rev), path_offset]) - # Update SVN properties based on source repo - sync_svn_props(source_url, source_rev, path_offset) + if options.keep_prop: + sync_svn_props(source_url, source_rev, path_offset) else: raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'" @@ -608,13 +612,12 @@ def process_svn_log_entry(log_entry, commit_paths, prefix = ""): source_url+"/"+path_offset+"@"+str(source_rev), path_offset]) def disp_svn_log_summary(log_entry): - ui.status("") + ui.status("------------------------------------------------------------------------") ui.status("r%s | %s | %s", log_entry['revision'], log_entry['author'], str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' '))) ui.status(log_entry['message']) - ui.status("------------------------------------------------------------------------") def real_main(args, parser): global source_url, target_url, rev_map @@ -645,6 +648,10 @@ def real_main(args, parser): parser.error("invalid end source revision value: %s" % (options.rev_end)) ui.status("Using source revision range %s:%s", source_start_rev, source_end_rev, level=ui.VERBOSE) + # TODO: If options.keep_date, should we try doing a "svn propset" on an *existing* revision + # as a sanity check, so we check if the pre-revprop-change hook script is correctly setup + # before doing first replay-commit? + target_end_rev = target_info['revision'] # Last revision # in the target repo wc_target = os.path.abspath('_wc_target') num_entries_proc = 0 @@ -676,6 +683,7 @@ def real_main(args, parser): # This is the revision we will start from for source_url source_start_rev = source_rev = int(source_start_log['revision']) ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE) + ui.status("") # For the initial commit to the target URL, export all the contents from # the source URL at the start-revision. @@ -700,7 +708,8 @@ def real_main(args, parser): # Update any properties on the newly added content paths = run_svn(["list", "--recursive", "-r", source_rev, source_url+"@"+str(source_rev)]) paths = paths.strip("\n").split("\n") - sync_svn_props(source_url, source_rev, "") + if options.keep_prop: + sync_svn_props(source_url, source_rev, "") for path in paths: if not path: continue @@ -708,7 +717,8 @@ def real_main(args, parser): path_is_dir = True if path[-1] == "/" else False path_offset = path.rstrip('/') if path_is_dir else path ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path_offset, level=ui.VERBOSE) - sync_svn_props(source_url, source_rev, path_offset) + if options.keep_prop: + sync_svn_props(source_url, source_rev, path_offset) # Commit the initial import num_entries_proc += 1 target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's @@ -723,10 +733,11 @@ def real_main(args, parser): # Re-build the rev_map based on any already-replayed history in target_url build_rev_map(target_url, target_end_rev, source_info) if not rev_map: - raise RuntimeError("Called with continue-mode, but no already-replayed history found in target repo: %s" % target_url) + parser.error("called with continue-mode, but no already-replayed source history found in target_url") source_start_rev = int(max(rev_map, key=rev_map.get)) assert source_start_rev ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE) + ui.status("") svn_vers_t = svnclient.get_svn_client_version() svn_vers = float(".".join(map(str, svn_vers_t[0:2]))) @@ -735,6 +746,9 @@ def real_main(args, parser): it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True) if source_start_rev < source_end_rev else [] source_rev = None + # TODO: Now that commit_from_svn_log_entry() might try to do a "svn propset svn:date", + # we might want some better KeyboardInterupt handilng here, to ensure that + # commit_from_svn_log_entry() always runs as an atomic unit. try: for log_entry in it_log_entries: if options.entries_proc_limit: @@ -782,13 +796,15 @@ def real_main(args, parser): def main(): # Defined as entry point. Must be callable without arguments. - usage = "Usage: %prog [OPTIONS] source_url target_url" + usage = "svn2svn, version %s\n" % str(full_version) + \ + " \n\n" + \ + "Usage: %prog [OPTIONS] source_url target_url\n" description = """\ - Replicate (replay) history from one SVN repository to another. Maintain - logical ancestry wherever possible, so that 'svn log' on the replayed - repo will correctly follow file/folder renames. +Replicate (replay) history from one SVN repository to another. Maintain +logical ancestry wherever possible, so that 'svn log' on the replayed repo +will correctly follow file/folder renames. - == Examples == +Examples: Create a copy of only /trunk from source repo, starting at r5000 $ svnadmin create /svn/target $ svn mkdir -m 'Add trunk' file:///svn/target/trunk @@ -812,27 +828,37 @@ def main(): logical ancestry where possible.""" parser = optparse.OptionParser(usage, description=description, formatter=HelpFormatter(), version="%prog "+str(full_version)) - #parser.remove_option("--help") - #parser.add_option("-h", "--help", dest="show_help", action="store_true", - # help="show this help message and exit") - parser.add_option("-r", "--revision", type="string", dest="svn_rev", metavar="ARG", - help="revision range to replay from source_url\n" + \ - "A revision argument can be one of:\n" + \ - " START start rev # (end will be 'HEAD')\n" + \ - " START:END start and ending rev #'s\n" + \ - "(Any revision # formats which SVN understands\n" + \ - " are supported, e.g. 'HEAD', '{2010-01-31}', etc.)") - parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author", default=False, - help="maintain original 'Author' info from source repo") + parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1, + help="enable additional output (use -vv or -vvv for more)") + parser.add_option("-a", "--archive", action="store_true", dest="archive", default=False, + help="archive/mirror mode; same as -UDP (see REQUIRE's below)\n" + "maintain same commit author, same commit time, and file/dir properties") + parser.add_option("-U", "--keep-author", action="store_true", dest="keep_author", default=False, + help="maintain same commit authors (svn:author) as source\n" + "(REQUIRES target_url be non-auth'd, e.g. file://-based, since this uses --username to set author)") + parser.add_option("-D", "--keep-date", action="store_true", dest="keep_date", default=False, + help="maintain same commit time (svn:date) as source\n" + "(REQUIRES 'pre-revprop-change' hook script to allow 'svn:date' changes)") + parser.add_option("-P", "--keep-prop", action="store_true", dest="keep_prop", default=False, + help="maintain same file/dir SVN properties as source") parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break", - help="continue from previous break") + help="continue from last source commit to target (based on svn2svn:* revprops)") + parser.add_option("-r", "--revision", type="string", dest="revision", metavar="ARG", + help="revision range to replay from source_url\n" + "A revision argument can be one of:\n" + " START start rev # (end will be 'HEAD')\n" + " START:END start and ending rev #'s\n" + "Any revision # formats which SVN understands are " + "supported, e.g. 'HEAD', '{2010-01-31}', etc.") + parser.add_option("-u", "--log-author", action="store_true", dest="log_author", default=False, + help="append source commit author to replayed commit mesages") + parser.add_option("-d", "--log-date", action="store_true", dest="log_date", default=False, + help="append source commit time to replayed commit messages") parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM", - help="maximum number of log entries to process") + help="maximum number of source revisions to process") parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False, - help="try processing next log entry but don't commit changes to " + help="process next source revision but don't commit changes to " "target working-copy (forces --limit=1)") - parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1, - help="enable additional output (use -vv or -vvv for more)") parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const", help="enable debugging output (same as -vvv)") global options @@ -858,6 +884,10 @@ def main(): rev = match.groups() options.rev_start = rev[0] if len(rev)>0 else None options.rev_end = rev[1] if len(rev)>1 else None + if options.archive: + options.keep_author = True + options.keep_date = True + options.keep_prop = True ui.update_config(options) return real_main(args, parser) diff --git a/svn2svn/svnclient.py b/svn2svn/svnclient.py index 858e99c..9761648 100644 --- a/svn2svn/svnclient.py +++ b/svn2svn/svnclient.py @@ -117,10 +117,8 @@ def parse_svn_log_xml(xml_string): date = entry.find('date') msg = entry.find('msg') d['author'] = author is not None and author.text or "No author" - if date is not None: - d['date'] = svn_date_to_timestamp(date.text) - else: - d['date'] = None + d['date_raw'] = date.text if date is not None else None + d['date'] = svn_date_to_timestamp(date.text) if date is not None else None d['message'] = msg is not None and msg.text and msg.text.replace('\r\n', '\n').replace('\n\r', '\n').replace('\r', '\n') or "" paths = [] for path in entry.findall('.//paths/path'): diff --git a/tests/make-replay-repo.sh b/tests/make-replay-repo.sh index 7843817..621e949 100755 --- a/tests/make-replay-repo.sh +++ b/tests/make-replay-repo.sh @@ -12,11 +12,14 @@ rm -rf $REPO _wc_target # Init repo echo "Creating _repo_replay..." svnadmin create $REPO +# Add pre-revprop-change hook script +cp ../hook-examples/pre-revprop-change_example.txt $REPO/hooks/pre-revprop-change +chmod 755 $REPO/hooks/pre-revprop-change echo "" ## svn2svn / -#../svn2svn.py -a $1 file://$PWD/_repo_ref file://$PWD/_repo_replay +#../svn2svn.py $* file://$PWD/_repo_ref file://$PWD/_repo_replay # svn2svn /trunk svn mkdir -q -m "Add /trunk" $REPOURL/trunk -../svn2svn.py -a $1 file://$PWD/_repo_ref/trunk file://$PWD/_repo_replay/trunk +../svn2svn.py $* file://$PWD/_repo_ref/trunk file://$PWD/_repo_replay/trunk -- 2.43.0