]> Tony Duckles's Git Repositories (git.nynim.org) - svn2svn.git/blob - svn2svn/run/svnreplay.py
svn2svn v1.7.0
[svn2svn.git] / svn2svn / run / svnreplay.py
1 """
2 Replicate (replay) changesets from one SVN repository to another.
3 """
4
5 from svn2svn import base_version, full_version
6 from svn2svn import ui
7 from svn2svn import shell
8 from svn2svn import svnclient
9 from svn2svn.shell import run_svn,run_shell_command
10 from svn2svn.errors import ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError
11 from svn2svn.run.common import in_svn, is_child_path, join_path, find_svn_ancestors
12 from parse import HelpFormatter
13 from breakhandler import BreakHandler
14
15 import sys
16 import os
17 import traceback
18 import operator
19 import optparse
20 import re
21 import urllib
22 from datetime import datetime
23
24 # Module-level variables/parameters
25 source_url = "" # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
26 source_repos_url = "" # URL to root of source SVN repo, e.g. 'http://server/svn/source'
27 source_base = "" # Relative path of source_url in source SVN repo, e.g. '/trunk'
28 source_repos_uuid = "" # UUID of source SVN repo
29 target_url ="" # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
30 target_repos_url = "" # URL to root of target SVN repo, e.g. 'http://server/svn/target'
31 target_base = "" # Relative path of target_url in target SVN repo, e.g. '/trunk'
32 rev_map = {} # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
33 options = None # optparser options
34
35 def parse_svn_commit_rev(output):
36 """
37 Parse the revision number from the output of "svn commit".
38 """
39 output_lines = output.strip("\n").split("\n")
40 rev_num = None
41 for line in output_lines:
42 if line[0:19] == 'Committed revision ':
43 rev_num = line[19:].rstrip('.')
44 break
45 assert rev_num is not None
46 return int(rev_num)
47
48 def commit_from_svn_log_entry(log_entry, commit_paths=None, target_revprops=None):
49 """
50 Given an SVN log entry and an optional list of changed paths, do an svn commit.
51 """
52 if options.beforecommit:
53 # Run optional external shell hook here, for doing pre-commit filtering
54 # $1 = Path to working copy
55 # $2 = Source revision #
56 args = [os.getcwd(), log_entry['revision']]
57 run_shell_command(options.beforecommit, args=args)
58 # Display the _wc_target "svn status" info if running in -vv (or higher) mode
59 if ui.get_level() >= ui.EXTRA:
60 ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
61 ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
62 # This will use the local timezone for displaying commit times
63 timestamp = int(log_entry['date'])
64 svn_date = str(datetime.fromtimestamp(timestamp))
65 # Uncomment this one one if you prefer UTC commit times
66 #svn_date = "%d 0" % timestamp
67 args = ["commit", "--force-log"]
68 message = log_entry['message']
69 if options.log_date:
70 message += "\nDate: " + svn_date
71 if options.log_author:
72 message += "\nAuthor: " + log_entry['author']
73 args += ["-m", message]
74 revprops = {}
75 if log_entry['revprops']:
76 # Carry forward any revprop's from the source revision
77 for v in log_entry['revprops']:
78 revprops[v['name']] = v['value']
79 if target_revprops:
80 # Add any extra revprop's we want to set for the target repo commits
81 for v in target_revprops:
82 revprops[v['name']] = v['value']
83 if revprops:
84 for key in revprops:
85 args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
86 if commit_paths:
87 if len(commit_paths)<100:
88 # If we don't have an excessive amount of individual changed paths, pass
89 # those to the "svn commit" command. Else, pass nothing so we commit at
90 # the root of the working-copy.
91 for c_path in commit_paths:
92 args += [svnclient.safe_path(c_path)]
93 rev_num = None
94 if not options.dry_run:
95 # Use BreakHandler class to temporarily redirect SIGINT handler, so that
96 # "svn commit" + post-commit rev-prop updating is a quasi-atomic unit.
97 # If user presses Ctrl-C during this, wait until after this full action
98 # has finished raising the KeyboardInterrupt exception.
99 bh = BreakHandler()
100 bh.enable()
101 # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
102 output = run_svn(args)
103 rev_num = parse_svn_commit_rev(output) if output else None
104 if rev_num is not None:
105 if options.keep_date:
106 run_svn(["propset", "--revprop", "-r", rev_num, "svn:date", log_entry['date_raw']])
107 if options.keep_author:
108 run_svn(["propset", "--revprop", "-r", rev_num, "svn:author", log_entry['author']])
109 ui.status("Committed revision %s (source r%s).", rev_num, log_entry['revision'])
110 bh.disable()
111 # Check if the user tried to press Ctrl-C
112 if bh.trapped:
113 raise KeyboardInterrupt
114 return rev_num
115
116 def verify_commit(source_rev, target_rev, log_entry=None):
117 """
118 Compare the ancestry/content/properties between source_url vs target_url
119 for a given revision.
120 """
121 error_cnt = 0
122 # Gather the offsets in the source repo to check
123 check_paths = []
124 remove_paths = []
125 # TODO: Need to make this ancestry aware
126 if options.verify == 1 and log_entry is not None: # Changed only
127 ui.status("Verifying source revision %s (only-changed)...", source_rev, level=ui.VERBOSE)
128 for d in log_entry['changed_paths']:
129 path = d['path']
130 if not is_child_path(path, source_base):
131 continue
132 if d['kind'] == "":
133 d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
134 assert (d['kind'] == 'file') or (d['kind'] == 'dir')
135 path_is_dir = True if d['kind'] == 'dir' else False
136 path_is_file = True if d['kind'] == 'file' else False
137 path_offset = path[len(source_base):].strip("/")
138 if d['action'] == 'D':
139 remove_paths.append(path_offset)
140 elif not path_offset in check_paths:
141 ui.status("verify_commit: path [mode=changed]: kind=%s: %s", d['kind'], path, level=ui.DEBUG, color='YELLOW')
142 if path_is_file:
143 ui.status(" "+"verify_commit [mode=changed]: check_paths.append('%s')", path_offset, level=ui.DEBUG, color='GREEN')
144 check_paths.append(path_offset)
145 if path_is_dir:
146 if not d['action'] in 'AR':
147 continue
148 child_paths = svnclient.list(source_url.rstrip("/")+"/"+path_offset, source_rev, recursive=True)
149 for p in child_paths:
150 child_path_is_dir = True if p['kind'] == 'dir' else False
151 child_path_offset = p['path']
152 if not child_path_is_dir:
153 # Only check files
154 working_path = (path_offset+"/" if path_offset else "") + child_path_offset
155 if not working_path in check_paths:
156 ui.status(" "+"verify_commit [mode=changed]: check_paths.append('%s'+'/'+'%s')", path_offset, child_path_offset, level=ui.DEBUG, color='GREEN')
157 check_paths.append(working_path)
158 if options.verify == 2: # All paths
159 ui.status("Verifying source revision %s (all)...", source_rev, level=ui.VERBOSE)
160 child_paths = svnclient.list(source_url, source_rev, recursive=True)
161 for p in child_paths:
162 child_path_is_dir = True if p['kind'] == 'dir' else False
163 child_path_offset = p['path']
164 if not child_path_is_dir:
165 # Only check files
166 ui.status("verify_commit [mode=all]: check_paths.append('%s')", child_path_offset, level=ui.DEBUG, color='GREEN')
167 check_paths.append(child_path_offset)
168
169 # If there were any paths deleted in the last revision (options.verify=1 mode),
170 # check that they were correctly deleted.
171 if remove_paths:
172 count_total = len(remove_paths)
173 count = 0
174 for path_offset in remove_paths:
175 count += 1
176 if in_svn(path_offset):
177 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA, color='RED')
178 ui.status("VerificationError: Path removed in source rev r%s, but still exists in target WC: %s", source_rev, path_offset, color='RED')
179 error_cnt +=1
180 else:
181 ui.status(" (%s/%s) Verify remove: OK: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA)
182
183 # Compare each of the check_path entries between source vs. target
184 if check_paths:
185 source_rev_first = int(min(rev_map, key=rev_map.get)) or 1 # The first source_rev we replayed into target
186 ui.status("verify_commit: source_rev_first:%s", source_rev_first, level=ui.DEBUG, color='YELLOW')
187 count_total = len(check_paths)
188 count = 0
189 for path_offset in check_paths:
190 count += 1
191 if count % 500 == 0:
192 ui.status("...processed %s (%s of %s)..." % (count, count, count_total), level=ui.VERBOSE)
193 ui.status("verify_commit: path_offset:%s", path_offset, level=ui.DEBUG, color='YELLOW')
194 source_log_entries = svnclient.run_svn_log(source_url.rstrip("/")+"/"+path_offset, source_rev, 1, source_rev-source_rev_first+1)
195 target_log_entries = svnclient.run_svn_log(target_url.rstrip("/")+"/"+path_offset, target_rev, 1, target_rev)
196 # Build a list of commits in source_log_entries which matches our
197 # target path_offset.
198 working_path = source_base+"/"+path_offset
199 source_revs = []
200 for log_entry in source_log_entries:
201 source_rev_tmp = log_entry['revision']
202 if source_rev_tmp < source_rev_first:
203 # Only process source revisions which have been replayed into target
204 break
205 #ui.status(" [verify_commit] source_rev_tmp:%s, working_path:%s\n%s", source_rev_tmp, working_path, pp.pformat(log_entry), level=ui.DEBUG, color='MAGENTA')
206 changed_paths_temp = []
207 for d in log_entry['changed_paths']:
208 path = d['path']
209 # Match working_path or any parents
210 if is_child_path(working_path, path):
211 ui.status(" verify_commit: changed_path: %s %s@%s (parent:%s)", d['action'], path, source_rev_tmp, working_path, level=ui.DEBUG, color='YELLOW')
212 changed_paths_temp.append({'path': path, 'data': d})
213 assert changed_paths_temp
214 # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
215 changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
216 # Find the action for our working_path in this revision. Use a loop to check in reverse order,
217 # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
218 working_path_next = working_path
219 match_d = {}
220 for v in changed_paths:
221 d = v['data']
222 if not match_d:
223 match_d = d
224 path = d['path']
225 if d['action'] not in svnclient.valid_svn_actions:
226 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
227 % (log_entry['revision'], d['action']))
228 if d['action'] in 'AR' and d['copyfrom_revision']:
229 # If we found a copy-from action for a parent path, adjust our
230 # working_path to follow the rename/copy-from, just like find_svn_ancestors().
231 working_path_next = working_path.replace(d['path'], d['copyfrom_path'])
232 match_d = d
233 break
234 if is_child_path(working_path, source_base):
235 # Only add source_rev's where the path changed in this revision was a child
236 # of source_base, so that we silently ignore any history that happened on
237 # non-source_base paths (e.g. ignore branch history if we're only replaying trunk).
238 is_diff = False
239 d = match_d
240 if d['action'] == 'M':
241 # For action="M", we need to throw out cases where the only change was to
242 # a property which we ignore, e.g. "svn:mergeinfo".
243 if d['kind'] == "":
244 d['kind'] = svnclient.get_kind(source_repos_url, working_path, log_entry['revision'], d['action'], log_entry['changed_paths'])
245 assert (d['kind'] == 'file') or (d['kind'] == 'dir')
246 if d['kind'] == 'file':
247 # Check for file-content changes
248 # TODO: This should be made ancestor-aware, since the file won't always be at the same path in rev-1
249 sum1 = run_shell_command("svn cat -r %s '%s' | md5sum" % (source_rev_tmp, source_repos_url+working_path+"@"+str(source_rev_tmp)))
250 sum2 = run_shell_command("svn cat -r %s '%s' | md5sum" % (source_rev_tmp-1, source_repos_url+working_path_next+"@"+str(source_rev_tmp-1)))
251 is_diff = True if sum1 <> sum2 else False
252 if not is_diff:
253 # Check for property changes
254 props1 = svnclient.propget_all(source_repos_url+working_path, source_rev_tmp)
255 props2 = svnclient.propget_all(source_repos_url+working_path_next, source_rev_tmp-1)
256 # Ignore changes to "svn:mergeinfo", since we don't copy that
257 if 'svn:mergeinfo' in props1: del props1['svn:mergeinfo']
258 if 'svn:mergeinfo' in props2: del props2['svn:mergeinfo']
259 for prop in props1:
260 if prop not in props2 or \
261 props1[prop] != props2[prop]:
262 is_diff = True
263 break
264 for prop in props2:
265 if prop not in props1 or \
266 props1[prop] != props2[prop]:
267 is_diff = True
268 break
269 if not is_diff:
270 ui.status(" verify_commit: skip %s@%s", working_path, source_rev_tmp, level=ui.DEBUG, color='GREEN_B', bold=True)
271 else:
272 is_diff = True
273 if is_diff:
274 ui.status(" verify_commit: source_revs.append(%s), working_path:%s", source_rev_tmp, working_path, level=ui.DEBUG, color='GREEN_B')
275 source_revs.append({'path': working_path, 'revision': source_rev_tmp})
276 working_path = working_path_next
277 # Build a list of all the target commits "svn log" returned
278 target_revs = []
279 target_revs_rmndr = []
280 for log_entry in target_log_entries:
281 target_rev_tmp = log_entry['revision']
282 ui.status(" verify_commit: target_revs.append(%s)", target_rev_tmp, level=ui.DEBUG, color='GREEN_B')
283 target_revs.append(target_rev_tmp)
284 target_revs_rmndr.append(target_rev_tmp)
285 # Compare the two lists
286 for d in source_revs:
287 working_path = d['path']
288 source_rev_tmp = d['revision']
289 target_rev_tmp = get_rev_map(source_rev_tmp, " ")
290 working_offset = working_path[len(source_base):].strip("/")
291 sum1 = run_shell_command("svn cat -r %s '%s' | md5sum" % (source_rev_tmp, source_repos_url+working_path+"@"+str(source_rev_tmp)))
292 sum2 = run_shell_command("svn cat -r %s '%s' | md5sum" % (target_rev_tmp, target_url+"/"+working_offset+"@"+str(target_rev_tmp))) if target_rev_tmp is not None else ""
293 #print "source@%s: %s" % (str(source_rev_tmp).ljust(6), sum1)
294 #print "target@%s: %s" % (str(target_rev_tmp).ljust(6), sum2)
295 ui.status(" verify_commit: %s: source=%s target=%s", working_offset, source_rev_tmp, target_rev_tmp, level=ui.DEBUG, color='GREEN')
296 if not target_rev_tmp:
297 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA, color='RED')
298 ui.status("VerificationError: Unable to find corresponding target_rev for source_rev r%s in rev_map (path_offset='%s')", source_rev_tmp, path_offset, color='RED')
299 error_cnt +=1
300 continue
301 if target_rev_tmp not in target_revs:
302 # If found a source_rev with no equivalent target_rev in target_revs,
303 # check if the only difference in source_rev vs. source_rev-1 is the
304 # removal/addition of a trailing newline char, since this seems to get
305 # stripped-out sometimes during the replay (via "svn export"?).
306 # Strip any trailing \r\n from file-content (http://stackoverflow.com/a/1656218/346778)
307 sum1 = run_shell_command("svn cat -r %s '%s' | perl -i -p0777we's/\\r\\n\z//' | md5sum" % (source_rev_tmp, source_repos_url+working_path+"@"+str(source_rev_tmp)))
308 sum2 = run_shell_command("svn cat -r %s '%s' | perl -i -p0777we's/\\r\\n\z//' | md5sum" % (source_rev_tmp-1, source_repos_url+working_path+"@"+str(source_rev_tmp-1)))
309 if sum1 <> sum2:
310 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA, color='RED')
311 ui.status("VerificationError: Found source_rev (r%s) with no corresponding target_rev: path_offset='%s'", source_rev_tmp, path_offset, color='RED')
312 error_cnt +=1
313 continue
314 target_revs_rmndr.remove(target_rev_tmp)
315 if target_revs_rmndr:
316 rmndr_list = ", ".join(map(str, target_revs_rmndr))
317 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA, color='RED')
318 ui.status("VerificationError: Found one or more *extra* target_revs: path_offset='%s', target_revs='%s'", path_offset, rmndr_list, color='RED')
319 error_cnt +=1
320 else:
321 ui.status(" (%s/%s) Verify path: OK: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA)
322
323 # Ensure there are no "extra" files in the target side
324 if options.verify == 2:
325 target_paths = []
326 child_paths = svnclient.list(target_url, target_rev, recursive=True)
327 for p in child_paths:
328 child_path_is_dir = True if p['kind'] == 'dir' else False
329 child_path_offset = p['path']
330 if not child_path_is_dir:
331 target_paths.append(child_path_offset)
332 # Compare
333 for path_offset in target_paths:
334 if not path_offset in check_paths:
335 ui.status("VerificationError: Path exists in target (@%s) but not source (@%s): %s", target_rev, source_rev, path_offset, color='RED')
336 error_cnt += 1
337 for path_offset in check_paths:
338 if not path_offset in target_paths:
339 ui.status("VerificationError: Path exists in source (@%s) but not target (@%s): %s", source_rev, target_rev, path_offset, color='RED')
340 error_cnt += 1
341
342 if error_cnt > 0:
343 raise VerificationError("Found %s verification errors" % (error_cnt))
344 ui.status("Verified revision %s (%s).", target_rev, "all" if options.verify == 2 else "only-changed")
345
346 def full_svn_revert():
347 """
348 Do an "svn revert" and proactively remove any extra files in the working copy.
349 """
350 run_svn(["revert", "--recursive", "."])
351 output = run_svn(["status"])
352 if output:
353 output_lines = output.strip("\n").split("\n")
354 for line in output_lines:
355 if line[0] == "?":
356 path = line[4:].strip(" ")
357 if os.path.isfile(path):
358 os.remove(path)
359 if os.path.isdir(path):
360 shell.rmtree(path)
361
362 def gen_tracking_revprops(source_rev):
363 """
364 Build an array of svn2svn-specific source-tracking revprops.
365 """
366 revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
367 {'name':'svn2svn:source_url', 'value':urllib.quote(source_url, ":/")},
368 {'name':'svn2svn:source_rev', 'value':source_rev}]
369 return revprops
370
371 def sync_svn_props(source_url, source_rev, path_offset):
372 """
373 Carry-forward any unversioned properties from the source repo to the
374 target WC.
375 """
376 source_props = svnclient.propget_all(join_path(source_url, path_offset), source_rev)
377 target_props = svnclient.propget_all(path_offset)
378 if 'svn:mergeinfo' in source_props:
379 # Never carry-forward "svn:mergeinfo"
380 del source_props['svn:mergeinfo']
381 for prop in target_props:
382 if prop not in source_props:
383 # Remove any properties which exist in target but not source
384 run_svn(["propdel", prop, svnclient.safe_path(path_offset)])
385 for prop in source_props:
386 if prop not in target_props or \
387 source_props[prop] != target_props[prop]:
388 # Set/update any properties which exist in source but not target or
389 # whose value differs between source vs. target.
390 run_svn(["propset", prop, source_props[prop], svnclient.safe_path(path_offset)])
391
392 def get_rev_map(source_rev, prefix):
393 """
394 Find the equivalent rev # in the target repo for the given rev # from the source repo.
395 """
396 ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
397 # Find the highest entry less-than-or-equal-to source_rev
398 for rev in range(int(source_rev), 0, -1):
399 in_rev_map = True if rev in rev_map else False
400 ui.status(prefix + ">> get_rev_map: rev=%s in_rev_map=%s", rev, str(in_rev_map), level=ui.DEBUG, color='BLACK_B')
401 if in_rev_map:
402 return int(rev_map[rev])
403 # Else, we fell off the bottom of the rev_map. Ruh-roh...
404 return None
405
406 def set_rev_map(source_rev, target_rev):
407 #ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
408 global rev_map
409 rev_map[int(source_rev)]=int(target_rev)
410
411 def build_rev_map(target_url, target_end_rev, source_info):
412 """
413 Check for any already-replayed history from source_url (source_info) and
414 build the mapping-table of source_rev -> target_rev.
415 """
416 global rev_map
417 rev_map = {}
418 ui.status("Rebuilding target_rev -> source_rev rev_map...", level=ui.VERBOSE)
419 proc_count = 0
420 it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
421 for log_entry in it_log_entries:
422 if log_entry['revprops']:
423 revprops = {}
424 for v in log_entry['revprops']:
425 if v['name'].startswith('svn2svn:'):
426 revprops[v['name']] = v['value']
427 if revprops and \
428 revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
429 revprops['svn2svn:source_url'] == urllib.quote(source_info['url'], ":/"):
430 source_rev = revprops['svn2svn:source_rev']
431 target_rev = log_entry['revision']
432 set_rev_map(source_rev, target_rev)
433 proc_count += 1
434 if proc_count % 500 == 0:
435 ui.status("...processed %s (%s of %s)..." % (proc_count, target_rev, target_end_rev), level=ui.VERBOSE)
436
437 def path_in_list(paths, path):
438 for p in paths:
439 if is_child_path(path, p):
440 return True
441 return False
442
443 def add_path(paths, path):
444 if not path_in_list(paths, path):
445 paths.append(path)
446
447 def in_ancestors(ancestors, ancestor):
448 match = True
449 for idx in range(len(ancestors)-1, 0, -1):
450 if int(ancestors[idx]['revision']) > ancestor['revision']:
451 match = is_child_path(ancestor['path'], ancestors[idx]['path'])
452 break
453 return match
454
455 def do_svn_add(source_url, path_offset, source_rev, source_ancestors, \
456 parent_copyfrom_path="", parent_copyfrom_rev="", \
457 export_paths={}, is_dir = False, skip_paths=[], prefix = ""):
458 """
459 Given the add'd source path, replay the "svn add/copy" commands to correctly
460 track renames across copy-from's.
461
462 For example, consider a sequence of events like this:
463 1. svn copy /trunk /branches/fix1
464 2. (Make some changes on /branches/fix1)
465 3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2 " Rename folder
466 4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt " Rename file inside renamed folder
467 5. svn co /trunk && svn merge /branches/fix1
468 After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
469 and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we just did
470 a straight "svn export+add" based on the /branches/fix1/Proj2 folder, we'd
471 lose the logical history that Proj2/file2.txt is really a descendant of
472 Proj1/file1.txt.
473
474 'path_offset' is the offset from source_base to the file to check ancestry for,
475 e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
476 'source_rev' is the revision ("svn log") that we're processing from the source repo.
477 'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
478 directory, when being called recursively by do_svn_add_dir().
479 'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
480 'is_dir' is whether path_offset is a directory (rather than a file).
481 """
482 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
483 ui.status(prefix + ">> do_svn_add: %s %s", join_path(source_base, path_offset)+"@"+str(source_rev),
484 " (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
485 level=ui.DEBUG, color='GREEN')
486 # Check if the given path has ancestors which chain back to the current source_base
487 found_ancestor = False
488 ancestors = find_svn_ancestors(source_repos_url, join_path(source_base, path_offset), source_rev, stop_base_path=source_base, prefix=prefix+" ")
489 ancestor = ancestors[len(ancestors)-1] if ancestors else None # Choose the eldest ancestor, i.e. where we reached stop_base_path=source_base
490 if ancestor and not in_ancestors(source_ancestors, ancestor):
491 ancestor = None
492 copyfrom_path = ancestor['copyfrom_path'] if ancestor else ""
493 copyfrom_rev = ancestor['copyfrom_rev'] if ancestor else ""
494 if ancestor:
495 # The copy-from path has ancestry back to source_url.
496 ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
497 level=ui.DEBUG, color='GREEN', bold=True)
498 found_ancestor = True
499 # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
500 # return None in the case where copyfrom_rev is *before* our source_start_rev.
501 tgt_rev = get_rev_map(copyfrom_rev, prefix+" ")
502 ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
503 else:
504 ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
505 found_ancestor = False
506 if found_ancestor and tgt_rev:
507 # Check if this path_offset in the target WC already has this ancestry, in which
508 # case there's no need to run the "svn copy" (again).
509 path_in_svn = in_svn(path_offset, prefix=prefix+" ")
510 log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+" ") else []
511 if (not log_entry or (log_entry['revision'] != tgt_rev)):
512 copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
513 ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
514 ui.status(prefix + " copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
515 ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
516 if path_in_svn and \
517 ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
518 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
519 # When being called recursively, if this child entry has the same ancestor as the
520 # the parent, then no need to try to run another "svn copy".
521 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
522 parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
523 pass
524 else:
525 # Copy this path from the equivalent path+rev in the target repo, to create the
526 # equivalent history.
527 if parent_copyfrom_path:
528 # If we have a parent copy-from path, we mis-match that so display a status
529 # message describing the action we're mimic'ing. If path_in_svn, then this
530 # is logically a "replace" rather than an "add".
531 ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), join_path(source_base, path_offset), ancestors[0]['copyfrom_path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
532 if path_in_svn:
533 # If local file is already under version-control, then this is a replace.
534 ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
535 svnclient.update(path_offset)
536 svnclient.remove(path_offset, force=True)
537 run_svn(["copy", "-r", tgt_rev, svnclient.safe_path(join_path(target_url, copyfrom_offset), tgt_rev), svnclient.safe_path(path_offset)])
538 if is_dir:
539 # Export the final verison of all files in this folder.
540 add_path(export_paths, path_offset)
541 else:
542 # Export the final verison of this file.
543 svnclient.export(source_repos_url+join_path(source_base, path_offset), source_rev, path_offset, force=True)
544 if options.keep_prop:
545 sync_svn_props(source_url, source_rev, path_offset)
546 else:
547 ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
548 else:
549 # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
550 # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
551 # Create (parent) directory if needed.
552 # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
553 # split-out to a shared tag?
554 p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or None
555 if p_path and not os.path.exists(p_path):
556 run_svn(["mkdir", svnclient.safe_path(p_path)])
557 if not in_svn(path_offset, prefix=prefix+" "):
558 if is_dir:
559 # Export the final verison of all files in this folder.
560 add_path(export_paths, path_offset)
561 else:
562 # Export the final verison of this file. We *need* to do this before running
563 # the "svn add", even if we end-up re-exporting this file again via export_paths.
564 svnclient.export(source_repos_url+join_path(source_base, path_offset), source_rev, path_offset, force=True)
565 # If not already under version-control, then "svn add" this file/folder.
566 run_svn(["add", "--parents", svnclient.safe_path(path_offset)])
567 if options.keep_prop:
568 sync_svn_props(source_url, source_rev, path_offset)
569 if is_dir:
570 # For any folders that we process, process any child contents, so that we correctly
571 # replay copies/replaces/etc.
572 do_svn_add_dir(source_url, path_offset, source_rev, source_ancestors,
573 copyfrom_path, copyfrom_rev, export_paths, skip_paths, prefix+" ")
574
575 def do_svn_add_dir(source_url, path_offset, source_rev, source_ancestors, \
576 parent_copyfrom_path, parent_copyfrom_rev, \
577 export_paths, skip_paths, prefix=""):
578 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
579 # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
580 # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
581 # associated remote repo folder. (Is this a problem?)
582 paths_local = svnclient.list(path_offset)
583 paths_remote = svnclient.list(join_path(source_url, path_offset), source_rev)
584 ui.status(prefix + ">> do_svn_add_dir: paths_local: %s", str(paths_local), level=ui.DEBUG, color='GREEN')
585 ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
586 # Update files/folders which exist in remote but not local
587 for p in paths_remote:
588 path_is_dir = True if p['kind'] == 'dir' else False
589 working_path = join_path(path_offset, p['path']).lstrip('/')
590 #print "working_path:%s = path_offset:%s + path:%s" % (working_path, path_offset, path)
591 if not working_path in skip_paths:
592 do_svn_add(source_url, working_path, source_rev, source_ancestors,
593 parent_copyfrom_path, parent_copyfrom_rev,
594 export_paths, path_is_dir, skip_paths, prefix+" ")
595 # Remove files/folders which exist in local but not remote
596 for p in paths_local:
597 if not p in paths_remote:
598 working_path = join_path(path_offset, p['path']).lstrip('/')
599 ui.status(" %s %s", 'D', join_path(source_base, working_path), level=ui.VERBOSE)
600 svnclient.update(working_path)
601 svnclient.remove(working_path, force=True)
602 # TODO: Does this handle deleted folders too? Wouldn't want to have a case
603 # where we only delete all files from folder but leave orphaned folder around.
604
605 def process_svn_log_entry(log_entry, ancestors, commit_paths, prefix = ""):
606 """
607 Process SVN changes from the given log entry. Build an array (commit_paths)
608 of the paths in the working-copy that were changed, i.e. the paths which
609 we'll pass to "svn commit".
610 """
611 export_paths = []
612 source_rev = log_entry['revision']
613 source_url = log_entry['url']
614 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
615 ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
616 for d in log_entry['changed_paths']:
617 # Get the full path for this changed_path
618 # e.g. '/branches/bug123/projectA/file1.txt'
619 path = d['path']
620 if not is_child_path(path, source_base):
621 # Ignore changed files that are not part of this subdir
622 ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
623 continue
624 if d['kind'] == "" or d['kind'] == 'none':
625 # The "kind" value was introduced in SVN 1.6, and "svn log --xml" won't return a "kind"
626 # value for commits made on a pre-1.6 repo, even if the server is now running 1.6.
627 # We need to use other methods to fetch the node-kind for these cases.
628 d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
629 assert (d['kind'] == 'file') or (d['kind'] == 'dir')
630 path_is_dir = True if d['kind'] == 'dir' else False
631 path_is_file = True if d['kind'] == 'file' else False
632 # Calculate the offset (based on source_base) for this changed_path
633 # e.g. 'projectA/file1.txt'
634 # (path = source_base + "/" + path_offset)
635 path_offset = path[len(source_base):].strip("/")
636 # Get the action for this path
637 action = d['action']
638 if action not in svnclient.valid_svn_actions:
639 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
640 % (source_rev, action))
641 ui.status(" %s %s%s", action, d['path'],
642 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
643 level=ui.VERBOSE)
644
645 # Try to be efficient and keep track of an explicit list of paths in the
646 # working copy that changed. If we commit from the root of the working copy,
647 # then SVN needs to crawl the entire working copy looking for pending changes.
648 commit_paths.append(path_offset)
649
650 # Special-handling for replace's
651 if action == 'R':
652 # If file was "replaced" (deleted then re-added, all in same revision),
653 # then we need to run the "svn rm" first, then change action='A'. This
654 # lets the normal code below handle re-"svn add"'ing the files. This
655 # should replicate the "replace".
656 if path_offset and in_svn(path_offset):
657 # Target path might not be under version-control yet, e.g. parent "add"
658 # was a copy-from a branch which had no ancestry back to trunk, and each
659 # child folder under that parent folder is a "replace" action on the final
660 # merge to trunk. Since the child folders will be in skip_paths, do_svn_add
661 # wouldn't have created them while processing the parent "add" path.
662 if path_is_dir:
663 # Need to "svn update" before "svn remove" in case child contents are at
664 # a higher rev than the (parent) path_offset.
665 svnclient.update(path_offset)
666 svnclient.remove(path_offset, force=True)
667 action = 'A'
668
669 # Handle all the various action-types
670 # (Handle "add" first, for "svn copy/move" support)
671 if action == 'A':
672 # Determine where to export from.
673 svn_copy = False
674 # Handle cases where this "add" was a copy from another URL in the source repo
675 if d['copyfrom_revision']:
676 copyfrom_path = d['copyfrom_path']
677 copyfrom_rev = d['copyfrom_revision']
678 skip_paths = []
679 for tmp_d in log_entry['changed_paths']:
680 tmp_path = tmp_d['path']
681 if is_child_path(tmp_path, path) and tmp_d['action'] in 'ARD':
682 # Build list of child entries which are also in the changed_paths list,
683 # so that do_svn_add() can skip processing these entries when recursing
684 # since we'll end-up processing them later. Don't include action="M" paths
685 # in this list because it's non-conclusive: it could just mean that the
686 # file was modified *after* the copy-from, so we still want do_svn_add()
687 # to re-create the correct ancestry.
688 tmp_path_offset = tmp_path[len(source_base):].strip("/")
689 skip_paths.append(tmp_path_offset)
690 do_svn_add(source_url, path_offset, source_rev, ancestors, "", "", export_paths, path_is_dir, skip_paths, prefix+" ")
691 # Else just "svn export" the files from the source repo and "svn add" them.
692 else:
693 # Create (parent) directory if needed
694 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or None
695 if p_path and not os.path.exists(p_path):
696 run_svn(["mkdir", svnclient.safe_path(p_path)])
697 # Export the entire added tree.
698 if path_is_dir:
699 # For directories, defer the (recurisve) "svn export". Might have a
700 # situation in a branch merge where the entry in the svn-log is a
701 # non-copy-from'd "add" but there are child contents (that we haven't
702 # gotten to yet in log_entry) that are copy-from's. When we try do
703 # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
704 # having pre-existing (svn-add'd) contents creates some trouble.
705 # Instead, just create the stub folders ("svn mkdir" above) and defer
706 # exporting the final file-state until the end.
707 add_path(export_paths, path_offset)
708 else:
709 # Export the final verison of this file. We *need* to do this before running
710 # the "svn add", even if we end-up re-exporting this file again via export_paths.
711 svnclient.export(join_path(source_url, path_offset), source_rev, path_offset, force=True)
712 if not in_svn(path_offset, prefix=prefix+" "):
713 # Need to use in_svn here to handle cases where client committed the parent
714 # folder and each indiv sub-folder.
715 run_svn(["add", "--parents", svnclient.safe_path(path_offset)])
716 if options.keep_prop:
717 sync_svn_props(source_url, source_rev, path_offset)
718
719 elif action == 'D':
720 if path_is_dir:
721 # For dirs, need to "svn update" before "svn remove" because the final
722 # "svn commit" will fail if the parent (path_offset) is at a lower rev
723 # than any of the child contents. This needs to be a recursive update.
724 svnclient.update(path_offset)
725 svnclient.remove(path_offset, force=True)
726
727 elif action == 'M':
728 if path_is_file:
729 svnclient.export(join_path(source_url, path_offset), source_rev, path_offset, force=True, non_recursive=True)
730 if path_is_dir:
731 # For dirs, need to "svn update" before export/prop-sync because the
732 # final "svn commit" will fail if the parent is at a lower rev than
733 # child contents. Just need to update the rev-state of the dir (d['path']),
734 # don't need to recursively update all child contents.
735 # (??? is this the right reason?)
736 svnclient.update(path_offset, non_recursive=True)
737 if options.keep_prop:
738 sync_svn_props(source_url, source_rev, path_offset)
739
740 else:
741 raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
742 % action)
743
744 # Export the final version of all add'd paths from source_url
745 if export_paths:
746 for path_offset in export_paths:
747 svnclient.export(join_path(source_url, path_offset), source_rev, path_offset, force=True)
748
749 def keep_revnum(source_rev, target_rev_last, wc_target_tmp):
750 """
751 Add "padding" target revisions as needed to keep source and target
752 revision #'s identical.
753 """
754 bh = BreakHandler()
755 if int(source_rev) <= int(target_rev_last):
756 raise InternalError("keep-revnum mode is enabled, "
757 "but source revision (r%s) is less-than-or-equal last target revision (r%s)" % \
758 (source_rev, target_rev_last))
759 if int(target_rev_last) < int(source_rev)-1:
760 # Add "padding" target revisions to keep source and target rev #'s identical
761 if os.path.exists(wc_target_tmp):
762 shell.rmtree(wc_target_tmp)
763 run_svn(["checkout", "-r", "HEAD", "--depth=empty", svnclient.safe_path(target_repos_url, "HEAD"), svnclient.safe_path(wc_target_tmp)])
764 for rev_num in range(int(target_rev_last)+1, int(source_rev)):
765 run_svn(["propset", "svn2svn:keep-revnum", rev_num, svnclient.safe_path(wc_target_tmp)])
766 # Prevent Ctrl-C's during this inner part, so we'll always display
767 # the "Commit revision ..." message if we ran a "svn commit".
768 bh.enable()
769 output = run_svn(["commit", "-m", "", svnclient.safe_path(wc_target_tmp)])
770 rev_num_tmp = parse_svn_commit_rev(output) if output else None
771 assert rev_num == rev_num_tmp
772 ui.status("Committed revision %s (keep-revnum).", rev_num)
773 bh.disable()
774 # Check if the user tried to press Ctrl-C
775 if bh.trapped:
776 raise KeyboardInterrupt
777 target_rev_last = rev_num
778 shell.rmtree(wc_target_tmp)
779 return target_rev_last
780
781 def disp_svn_log_summary(log_entry):
782 ui.status("------------------------------------------------------------------------", level=ui.VERBOSE)
783 ui.status("r%s | %s | %s",
784 log_entry['revision'],
785 log_entry['author'],
786 str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')), level=ui.VERBOSE)
787 ui.status(log_entry['message'], level=ui.VERBOSE)
788
789 def real_main(args):
790 global source_url, target_url, rev_map
791 # Use urllib.unquote() to URL-decode source_url/target_url values.
792 # All URLs passed to run_svn() should go through svnclient.safe_path()
793 # and we don't want to end-up *double* urllib.quote'ing if the user-
794 # supplied source/target URL's are already URL-encoded.
795 source_url = urllib.unquote(args.pop(0).rstrip("/")) # e.g. 'http://server/svn/source/trunk'
796 target_url = urllib.unquote(args.pop(0).rstrip("/")) # e.g. 'file:///svn/target/trunk'
797 ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
798
799 # Make sure that both the source and target URL's are valid
800 source_info = svnclient.info(source_url)
801 assert is_child_path(source_url, source_info['repos_url'])
802 target_info = svnclient.info(target_url)
803 assert is_child_path(target_url, target_info['repos_url'])
804
805 # Init global vars
806 global source_repos_url,source_base,source_repos_uuid
807 source_repos_url = source_info['repos_url'] # e.g. 'http://server/svn/source'
808 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
809 source_repos_uuid = source_info['repos_uuid']
810 global target_repos_url,target_base
811 target_repos_url = target_info['repos_url'] # e.g. 'http://server/svn/target'
812 target_base = target_url[len(target_repos_url):] # e.g. '/trunk'
813
814 # Init start and end revision
815 try:
816 source_start_rev = svnclient.get_rev(source_repos_url, options.rev_start if options.rev_start else 1)
817 except ExternalCommandFailed:
818 print "Error: Invalid start source revision value: %s" % (options.rev_start)
819 return 1
820 try:
821 source_end_rev = svnclient.get_rev(source_repos_url, options.rev_end if options.rev_end else "HEAD")
822 except ExternalCommandFailed:
823 print "Error: Invalid end source revision value: %s" % (options.rev_end)
824 return 1
825 ui.status("Using source revision range %s:%s", source_start_rev, source_end_rev, level=ui.VERBOSE)
826
827 # TODO: If options.keep_date, should we try doing a "svn propset" on an *existing* revision
828 # as a sanity check, so we check if the pre-revprop-change hook script is correctly setup
829 # before doing first replay-commit?
830
831 target_rev_last = target_info['revision'] # Last revision # in the target repo
832 wc_target = os.path.abspath('_wc_target')
833 wc_target_tmp = os.path.abspath('_wc_target_tmp')
834 num_entries_proc = 0
835 commit_count = 0
836 source_rev = None
837 target_rev = None
838
839 # Check out a working copy of target_url if needed
840 wc_exists = os.path.exists(wc_target)
841 if wc_exists and not options.cont_from_break:
842 shell.rmtree(wc_target)
843 wc_exists = False
844 if not wc_exists:
845 ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
846 svnclient.svn_checkout(target_url, wc_target)
847 os.chdir(wc_target)
848 if wc_exists:
849 # If using an existing WC, make sure it's clean ("svn revert")
850 ui.status("Cleaning-up _wc_target...", level=ui.VERBOSE)
851 run_svn(["cleanup"])
852 full_svn_revert()
853
854 if not options.cont_from_break:
855 # Warn user if trying to start (non-continue) into a non-empty target path
856 if not options.force_nocont:
857 top_paths = svnclient.list(target_url, "HEAD")
858 if len(top_paths)>0:
859 print "Error: Trying to replay (non-continue-mode) into a non-empty target_url location. " \
860 "Use --force if you're sure this is what you want."
861 return 1
862 # Get the first log entry at/after source_start_rev, which is where
863 # we'll do the initial import from.
864 source_ancestors = find_svn_ancestors(source_repos_url, source_base, source_end_rev, prefix=" ")
865 it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False, ancestors=source_ancestors)
866 source_start_log = None
867 for log_entry in it_log_start:
868 # Pick the first entry. Need to use a "for ..." loop since we're using an iterator.
869 source_start_log = log_entry
870 break
871 if not source_start_log:
872 raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
873 (source_start_rev, source_end_rev, source_url))
874
875 # This is the revision we will start from for source_url
876 source_start_rev = int(source_start_log['revision'])
877 ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
878 ui.status("", level=ui.VERBOSE)
879 if options.keep_revnum and source_rev > target_rev_last:
880 target_rev_last = keep_revnum(source_rev, target_rev_last, wc_target_tmp)
881
882 # For the initial commit to the target URL, export all the contents from
883 # the source URL at the start-revision.
884 disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_repos_url, source_start_rev, source_start_rev))
885 # Export and add file-contents from source_url@source_start_rev
886 source_start_url = source_url if not source_ancestors else source_repos_url+source_ancestors[len(source_ancestors)-1]['copyfrom_path']
887 top_paths = svnclient.list(source_start_url, source_start_rev)
888 for p in top_paths:
889 # For each top-level file/folder...
890 path_is_dir = True if p['kind'] == "dir" else False
891 path_offset = p['path']
892 if in_svn(path_offset, prefix=" "):
893 raise InternalError("Cannot replay history on top of pre-existing structure: %s" % join_path(source_start_url, path_offset))
894 if path_is_dir and not os.path.exists(path_offset):
895 os.makedirs(path_offset)
896 svnclient.export(join_path(source_start_url, path_offset), source_start_rev, path_offset, force=True)
897 run_svn(["add", svnclient.safe_path(path_offset)])
898 # Update any properties on the newly added content
899 paths = svnclient.list(source_start_url, source_start_rev, recursive=True)
900 if options.keep_prop:
901 sync_svn_props(source_start_url, source_start_rev, "")
902 for p in paths:
903 path_offset = p['path']
904 ui.status(" A %s", join_path(source_base, path_offset), level=ui.VERBOSE)
905 if options.keep_prop:
906 sync_svn_props(source_start_url, source_start_rev, path_offset)
907 # Commit the initial import
908 num_entries_proc += 1
909 target_revprops = gen_tracking_revprops(source_start_rev) # Build source-tracking revprop's
910 target_rev = commit_from_svn_log_entry(source_start_log, target_revprops=target_revprops)
911 if target_rev:
912 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
913 set_rev_map(source_start_rev, target_rev)
914 commit_count += 1
915 target_rev_last = target_rev
916 if options.verify:
917 verify_commit(source_rev, target_rev_last)
918 else:
919 # Re-build the rev_map based on any already-replayed history in target_url
920 build_rev_map(target_url, target_rev_last, source_info)
921 if not rev_map:
922 print "Error: Called with continue-mode, but no already-replayed source history found in target_url."
923 return 1
924 source_start_rev = int(max(rev_map, key=rev_map.get))
925 assert source_start_rev
926 ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
927 ui.status("", level=ui.VERBOSE)
928
929 svn_vers_t = svnclient.version()
930 svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
931
932 # Load SVN log starting from source_start_rev + 1
933 source_ancestors = find_svn_ancestors(source_repos_url, source_base, source_end_rev, prefix=" ")
934 it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True, ancestors=source_ancestors) if source_start_rev < source_end_rev else []
935 source_rev_last = source_start_rev
936 exit_code = 0
937
938 try:
939 for log_entry in it_log_entries:
940 if options.entries_proc_limit:
941 if num_entries_proc >= options.entries_proc_limit:
942 break
943 # Replay this revision from source_url into target_url
944 source_rev = log_entry['revision']
945 log_url = log_entry['url']
946 #print "source_url:%s log_url:%s" % (source_url, log_url)
947 if options.keep_revnum:
948 if source_rev < target_rev_last:
949 print "Error: Last target revision (r%s) is equal-or-higher than starting source revision (r%s). " \
950 "Cannot use --keep-revnum mode." % (target_rev_last, source_start_rev)
951 return 1
952 target_rev_last = keep_revnum(source_rev, target_rev_last, wc_target_tmp)
953 disp_svn_log_summary(log_entry)
954 # Process all the changed-paths in this log entry
955 commit_paths = []
956 process_svn_log_entry(log_entry, source_ancestors, commit_paths)
957 num_entries_proc += 1
958 # Commit any changes made to _wc_target
959 target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's
960 target_rev = commit_from_svn_log_entry(log_entry, commit_paths, target_revprops=target_revprops)
961 source_rev_last = source_rev
962 if target_rev:
963 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
964 source_rev = log_entry['revision']
965 set_rev_map(source_rev, target_rev)
966 target_rev_last = target_rev
967 commit_count += 1
968 if options.verify:
969 verify_commit(source_rev, target_rev_last, log_entry)
970 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
971 if svn_vers >= 1.7 and (commit_count % 100 == 0):
972 run_svn(["cleanup"])
973 if source_rev_last == source_start_rev:
974 # If there were no new source_url revisions to process, still trigger
975 # "full-mode" verify check (if enabled).
976 if options.verify:
977 verify_commit(source_rev_last, target_rev_last)
978
979 except KeyboardInterrupt:
980 exit_code = 1
981 print "\nStopped by user."
982 print "\nCleaning-up..."
983 run_svn(["cleanup"])
984 full_svn_revert()
985 except:
986 exit_code = 1
987 print "\nCommand failed with following error:\n"
988 traceback.print_exc()
989 print "\nCleaning-up..."
990 run_svn(["cleanup"])
991 print run_svn(["status"])
992 full_svn_revert()
993 finally:
994 print "\nFinished at source revision %s%s." % (source_rev_last, " (dry-run)" if options.dry_run else "")
995
996 return exit_code
997
998 def main():
999 # Defined as entry point. Must be callable without arguments.
1000 usage = "svn2svn, version %s\n" % str(full_version) + \
1001 "<http://nynim.org/projects/svn2svn> <https://github.com/tonyduckles/svn2svn>\n\n" + \
1002 "Usage: %prog [OPTIONS] source_url target_url\n"
1003 description = """\
1004 Replicate (replay) history from one SVN repository to another. Maintain
1005 logical ancestry wherever possible, so that 'svn log' on the replayed repo
1006 will correctly follow file/folder renames.
1007
1008 Examples:
1009 Create a copy of only /trunk from source repo, starting at r5000
1010 $ svnadmin create /svn/target
1011 $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
1012 $ svnreplay -av -r 5000 http://server/source/trunk file:///svn/target/trunk
1013 1. The target_url will be checked-out to ./_wc_target
1014 2. The first commit to http://server/source/trunk at/after r5000 will be
1015 exported & added into _wc_target
1016 3. All revisions affecting http://server/source/trunk (starting at r5000)
1017 will be replayed to _wc_target. Any add/copy/move/replaces that are
1018 copy-from'd some path outside of /trunk (e.g. files renamed on a
1019 /branch and branch was merged into /trunk) will correctly maintain
1020 logical ancestry where possible.
1021
1022 Use continue-mode (-c) to pick-up where the last run left-off
1023 $ svnreplay -avc http://server/source/trunk file:///svn/target/trunk
1024 1. The target_url will be checked-out to ./_wc_target, if not already
1025 checked-out
1026 2. All new revisions affecting http://server/source/trunk starting from
1027 the last replayed revision to file:///svn/target/trunk (based on the
1028 svn2svn:* revprops) will be replayed to _wc_target, maintaining all
1029 logical ancestry where possible."""
1030 parser = optparse.OptionParser(usage, description=description,
1031 formatter=HelpFormatter(), version="%prog "+str(full_version))
1032 parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
1033 help="Enable additional output (use -vv or -vvv for more).")
1034 parser.add_option("-a", "--archive", action="store_true", dest="archive", default=False,
1035 help="Archive/mirror mode; same as -UDP (see REQUIRES below).\n"
1036 "Maintain same commit author, same commit time, and file/dir properties.")
1037 parser.add_option("-U", "--keep-author", action="store_true", dest="keep_author", default=False,
1038 help="Maintain same commit authors (svn:author) as source.\n"
1039 "(REQUIRES 'pre-revprop-change' hook script to allow 'svn:author' changes.)")
1040 parser.add_option("-D", "--keep-date", action="store_true", dest="keep_date", default=False,
1041 help="Maintain same commit time (svn:date) as source.\n"
1042 "(REQUIRES 'pre-revprop-change' hook script to allow 'svn:date' changes.)")
1043 parser.add_option("-P", "--keep-prop", action="store_true", dest="keep_prop", default=False,
1044 help="Maintain same file/dir SVN properties as source.")
1045 parser.add_option("-R", "--keep-revnum", action="store_true", dest="keep_revnum", default=False,
1046 help="Maintain same rev #'s as source. creates placeholder target "
1047 "revisions (by modifying a 'svn2svn:keep-revnum' property at the root of the target repo).")
1048 parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
1049 help="Continue from last source commit to target (based on svn2svn:* revprops).")
1050 parser.add_option("-f", "--force", action="store_true", dest="force_nocont",
1051 help="Allow replaying into a non-empty target folder.")
1052 parser.add_option("-r", "--revision", type="string", dest="revision", metavar="ARG",
1053 help="Revision range to replay from source_url.\n"
1054 "A revision argument can be one of:\n"
1055 " START Start rev # (end will be 'HEAD')\n"
1056 " START:END Start and ending rev #'s\n"
1057 "Any revision # formats which SVN understands are "
1058 "supported, e.g. 'HEAD', '{2010-01-31}', etc.")
1059 parser.add_option("-u", "--log-author", action="store_true", dest="log_author", default=False,
1060 help="Append source commit author to replayed commit mesages.")
1061 parser.add_option("-d", "--log-date", action="store_true", dest="log_date", default=False,
1062 help="Append source commit time to replayed commit messages.")
1063 parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
1064 help="Maximum number of source revisions to process.")
1065 parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
1066 help="Process next source revision but don't commit changes to "
1067 "target working-copy (forces --limit=1).")
1068 parser.add_option("-x", "--verify", action="store_const", const=1, dest="verify",
1069 help="Verify ancestry and content for changed paths in commit after every target commit or last target commit.")
1070 parser.add_option("-X", "--verify-all", action="store_const", const=2, dest="verify",
1071 help="Verify ancestry and content for entire target_url tree after every target commit or last target commit.")
1072 parser.add_option("--pre-commit", type="string", dest="beforecommit", metavar="CMD",
1073 help="Run the given shell script before each replayed commit, e.g. "
1074 "to modify file-content during replay.\n"
1075 "Called as: CMD [wc_path] [source_rev]")
1076 parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
1077 help="Enable debugging output (same as -vvv).")
1078 global options
1079 options, args = parser.parse_args()
1080 if len(args) != 2:
1081 parser.error("incorrect number of arguments")
1082 if options.verbosity < 10:
1083 # Expand multiple "-v" arguments to a real ui._level value
1084 options.verbosity *= 10
1085 if options.dry_run:
1086 # When in dry-run mode, only try to process the next log_entry
1087 options.entries_proc_limit = 1
1088 options.rev_start = None
1089 options.rev_end = None
1090 if options.revision:
1091 # Reg-ex for matching a revision arg (http://svnbook.red-bean.com/en/1.5/svn.tour.revs.specifiers.html#svn.tour.revs.dates)
1092 rev_patt = '[0-9A-Z]+|\{[0-9A-Za-z/\\ :-]+\}'
1093 rev = None
1094 match = re.match('^('+rev_patt+'):('+rev_patt+')$', options.revision) # First try start:end match
1095 if match is None: match = re.match('^('+rev_patt+')$', options.revision) # Next, try start match
1096 if match is None:
1097 parser.error("unexpected --revision argument format; see 'svn help log' for valid revision formats")
1098 rev = match.groups()
1099 options.rev_start = rev[0] if len(rev)>0 else None
1100 options.rev_end = rev[1] if len(rev)>1 else None
1101 if options.archive:
1102 options.keep_author = True
1103 options.keep_date = True
1104 options.keep_prop = True
1105 ui.update_config(options)
1106 return real_main(args)
1107
1108
1109 if __name__ == "__main__":
1110 sys.exit(main() or 0)