]> Tony Duckles's Git Repositories (git.nynim.org) - svn2svn.git/blob - svn2svn/run/svn2svn.py
Add --keep-date, --keep-prop, --archive, --log-author, --log-date command line args
[svn2svn.git] / svn2svn / run / svn2svn.py
1 """
2 Replicate (replay) changesets from one SVN repository to another.
3 """
4
5 from .. import base_version, full_version
6 from .. import ui
7 from .. import svnclient
8 from ..shell import run_svn
9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
10 from parse import HelpFormatter
11
12 import sys
13 import os
14 import time
15 import traceback
16 import shutil
17 import operator
18 import optparse
19 import re
20 from datetime import datetime
21
22 _valid_svn_actions = "MARD" # The list of known SVN action abbr's, from "svn log"
23
24 # Module-level variables/parameters
25 source_url = "" # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
26 source_repos_url = "" # URL to root of source SVN repo, e.g. 'http://server/svn/source'
27 source_base = "" # Relative path of source_url in source SVN repo, e.g. '/trunk'
28 source_repos_uuid = "" # UUID of source SVN repo
29 target_url ="" # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
30 rev_map = {} # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
31 options = None # optparser options
32
33 def commit_from_svn_log_entry(log_entry, commit_paths=None, target_revprops=None):
34 """
35 Given an SVN log entry and an optional list of changed paths, do an svn commit.
36 """
37 # TODO: Run optional external shell hook here, for doing pre-commit filtering
38 # Display the _wc_target "svn status" info if running in -vv (or higher) mode
39 if ui.get_level() >= ui.EXTRA:
40 ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
41 ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
42 # This will use the local timezone for displaying commit times
43 timestamp = int(log_entry['date'])
44 svn_date = str(datetime.fromtimestamp(timestamp))
45 # Uncomment this one one if you prefer UTC commit times
46 #svn_date = "%d 0" % timestamp
47 args = ["commit", "--force-log"]
48 message = log_entry['message']
49 if options.log_date:
50 message += "\nDate: " + svn_date
51 if options.log_author:
52 message += "\nAuthor: " + log_entry['author']
53 if options.keep_author:
54 args += ["--username", log_entry['author']]
55 args += ["-m", message]
56 revprops = {}
57 if log_entry['revprops']:
58 # Carry forward any revprop's from the source revision
59 for v in log_entry['revprops']:
60 revprops[v['name']] = v['value']
61 if target_revprops:
62 # Add any extra revprop's we want to set for the target repo commits
63 for v in target_revprops:
64 revprops[v['name']] = v['value']
65 if revprops:
66 for key in revprops:
67 args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
68 if commit_paths:
69 if len(commit_paths)<100:
70 # If we don't have an excessive amount of individual changed paths, pass
71 # those to the "svn commit" command. Else, pass nothing so we commit at
72 # the root of the working-copy.
73 args += list(commit_paths)
74 rev = None
75 if not options.dry_run:
76 # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
77 output = run_svn(args)
78 if output:
79 output_lines = output.strip("\n").split("\n")
80 rev = ""
81 for line in output_lines:
82 if line[0:19] == 'Committed revision ':
83 rev = line[19:].rstrip('.')
84 break
85 if rev:
86 ui.status("Committed revision %s.", rev)
87 return rev
88
89 def full_svn_revert():
90 """
91 Do an "svn revert" and proactively remove any extra files in the working copy.
92 """
93 run_svn(["revert", "--recursive", "."])
94 output = run_svn(["status"])
95 if output:
96 output_lines = output.strip("\n").split("\n")
97 for line in output_lines:
98 if line[0] == "?":
99 path = line[4:].strip(" ")
100 if os.path.isfile(path):
101 os.remove(path)
102 if os.path.isdir(path):
103 shutil.rmtree(path)
104
105 def gen_tracking_revprops(source_rev):
106 """
107 Build an array of svn2svn-specific source-tracking revprops.
108 """
109 revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
110 {'name':'svn2svn:source_url', 'value':source_url},
111 {'name':'svn2svn:source_rev', 'value':source_rev}]
112 return revprops
113
114 def sync_svn_props(source_url, source_rev, path_offset):
115 """
116 Carry-forward any unversioned properties from the source repo to the
117 target WC.
118 """
119 source_props = svnclient.get_all_props(source_url+"/"+path_offset, source_rev)
120 target_props = svnclient.get_all_props(path_offset)
121 if 'svn:mergeinfo' in source_props:
122 # Never carry-forward "svn:mergeinfo"
123 del source_props['svn:mergeinfo']
124 for prop in target_props:
125 if prop not in source_props:
126 # Remove any properties which exist in target but not source
127 run_svn(["propdel", prop, path_offset])
128 for prop in source_props:
129 if prop not in target_props or \
130 source_props[prop] != target_props[prop]:
131 # Set/update any properties which exist in source but not target or
132 # whose value differs between source vs. target.
133 run_svn(["propset", prop, source_props[prop], path_offset])
134
135 def in_svn(p, require_in_repo=False, prefix=""):
136 """
137 Check if a given file/folder is being tracked by Subversion.
138 Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
139 With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
140 Use "svn status" to check the status of the file/folder.
141 """
142 entries = svnclient.get_svn_status(p, no_recursive=True)
143 if not entries:
144 return False
145 d = entries[0]
146 if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
147 # If caller requires this path to be in the SVN repo, prevent returning True
148 # for paths that are only locally-added.
149 ret = False
150 else:
151 # Don't consider files tracked as deleted in the WC as under source-control.
152 # Consider files which are locally added/copied as under source-control.
153 ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
154 ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
155 return ret
156
157 def is_child_path(path, p_path):
158 return True if (path == p_path) or (path.startswith(p_path+"/")) else False
159
160 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
161 """
162 Given a source path, walk the SVN history backwards to inspect the ancestory of
163 that path, seeing if it traces back to base_path. Build an array of copyfrom_path
164 and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
165 which base_path is a substring match of (e.g. we crawled back to the initial branch-
166 copy from trunk), then return the collection of ancestor paths. Otherwise,
167 copyfrom_path has no ancestory compared to base_path.
168
169 This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
170 file/folder was renamed in a branch and then that branch was merged back to trunk.
171
172 'svn_repos_url' is the full URL to the root of the SVN repository,
173 e.g. 'file:///path/to/repo'
174 'base_path' is the path in the SVN repo to the target path we're trying to
175 trace ancestry back to, e.g. 'trunk'.
176 'source_path' is the path in the SVN repo to the source path to start checking
177 ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
178 (full_path = svn_repos_url+base_path+"/"+path_offset)
179 'source_rev' is the revision to start walking the history of source_path backwards from.
180 """
181 ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s base_path: %s",
182 svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
183 done = False
184 working_path = base_path+"/"+source_path
185 working_rev = source_rev
186 first_iter_done = False
187 ancestors_temp = []
188 while not done:
189 # Get the first "svn log" entry for this path (relative to @rev)
190 ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
191 log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path, 1, working_rev, True)
192 if not log_entry:
193 ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
194 done = True
195 break
196 # If we found a copy-from case which matches our base_path, we're done.
197 # ...but only if we've at least tried to search for the first copy-from path.
198 if first_iter_done and is_child_path(working_path, base_path):
199 ui.status(prefix + ">> find_svn_ancestors: Done: Found is_child_path(working_path, base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
200 done = True
201 break
202 first_iter_done = True
203 # Search for any actions on our target path (or parent paths).
204 changed_paths_temp = []
205 for d in log_entry['changed_paths']:
206 path = d['path']
207 if path in working_path:
208 changed_paths_temp.append({'path': path, 'data': d})
209 if not changed_paths_temp:
210 # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
211 ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
212 done = True
213 continue
214 # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
215 changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
216 # Find the action for our working_path in this revision. Use a loop to check in reverse order,
217 # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
218 for v in changed_paths:
219 d = v['data']
220 path = d['path']
221 # Check action-type for this file
222 action = d['action']
223 if action not in _valid_svn_actions:
224 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
225 % (log_entry['revision'], action))
226 ui.status(prefix + "> %s %s%s", action, path,
227 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
228 level=ui.DEBUG, color='YELLOW')
229 if action == 'D':
230 # If file/folder was deleted, it has no ancestor
231 ancestors_temp = []
232 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
233 done = True
234 break
235 if action in 'RA':
236 # If file/folder was added/replaced but not a copy, it has no ancestor
237 if not d['copyfrom_path']:
238 ancestors_temp = []
239 ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
240 "Added" if action == "A" else "Replaced",
241 level=ui.DEBUG, color='YELLOW')
242 done = True
243 break
244 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
245 # and keep checking for ancestors
246 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
247 action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
248 level=ui.DEBUG, color='YELLOW')
249 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
250 'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
251 working_path = working_path.replace(d['path'], d['copyfrom_path'])
252 working_rev = d['copyfrom_revision']
253 # Follow the copy and keep on searching
254 break
255 ancestors = []
256 if ancestors_temp:
257 ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
258 working_path = base_path+"/"+source_path
259 for idx in range(len(ancestors_temp)):
260 d = ancestors_temp[idx]
261 working_path = working_path.replace(d['path'], d['copyfrom_path'])
262 working_rev = d['copyfrom_rev']
263 ancestors.append({'path': working_path, 'revision': working_rev})
264 if ui.get_level() >= ui.DEBUG:
265 max_len = 0
266 for idx in range(len(ancestors)):
267 d = ancestors[idx]
268 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
269 ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
270 for idx in range(len(ancestors)-1):
271 d = ancestors[idx]
272 d_next = ancestors[idx+1]
273 ui.status(prefix + " [%s] %s <-- %s", idx,
274 str(d['path']+"@"+str(d['revision'])).ljust(max_len),
275 str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
276 level=ui.DEBUG, color='YELLOW')
277 else:
278 ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
279 svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
280 return ancestors
281
282 def get_rev_map(source_rev, prefix):
283 """
284 Find the equivalent rev # in the target repo for the given rev # from the source repo.
285 """
286 ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
287 # Find the highest entry less-than-or-equal-to source_rev
288 for rev in range(int(source_rev), 0, -1):
289 ui.status(prefix + ">> get_rev_map: rev=%s in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
290 if rev in rev_map:
291 return int(rev_map[rev])
292 # Else, we fell off the bottom of the rev_map. Ruh-roh...
293 return None
294
295 def set_rev_map(source_rev, target_rev):
296 ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
297 global rev_map
298 rev_map[int(source_rev)]=int(target_rev)
299
300 def build_rev_map(target_url, target_end_rev, source_info):
301 """
302 Check for any already-replayed history from source_url (source_info) and
303 build the mapping-table of source_rev -> target_rev.
304 """
305 global rev_map
306 rev_map = {}
307 ui.status("Rebuilding target_rev -> source_rev rev_map...", level=ui.VERBOSE)
308 proc_count = 0
309 it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
310 for log_entry in it_log_entries:
311 if log_entry['revprops']:
312 revprops = {}
313 for v in log_entry['revprops']:
314 if v['name'].startswith('svn2svn:'):
315 revprops[v['name']] = v['value']
316 if revprops and \
317 revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
318 revprops['svn2svn:source_url'] == source_info['url']:
319 source_rev = revprops['svn2svn:source_rev']
320 target_rev = log_entry['revision']
321 set_rev_map(source_rev, target_rev)
322
323 def get_svn_dirlist(svn_path, rev_number = ""):
324 """
325 Get a list of all the child contents (recusive) of the given folder path.
326 """
327 args = ["list"]
328 path = svn_path
329 if rev_number:
330 args += ["-r", rev_number]
331 path += "@"+str(rev_number)
332 args += [path]
333 paths = run_svn(args, no_fail=True)
334 paths = paths.strip("\n").split("\n") if len(paths)>1 else []
335 return paths
336
337 def path_in_list(paths, path):
338 for p in paths:
339 if is_child_path(path, p):
340 return True
341 return False
342
343 def add_path(paths, path):
344 if not path_in_list(paths, path):
345 paths.append(path)
346
347 def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom_rev="", \
348 export_paths={}, is_dir = False, skip_paths=[], prefix = ""):
349 """
350 Given the add'd source path, replay the "svn add/copy" commands to correctly
351 track renames across copy-from's.
352
353 For example, consider a sequence of events like this:
354 1. svn copy /trunk /branches/fix1
355 2. (Make some changes on /branches/fix1)
356 3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2 " Rename folder
357 4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt " Rename file inside renamed folder
358 5. svn co /trunk && svn merge /branches/fix1
359 After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
360 and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
361 to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
362 we'd lose the logical history that Proj2/file2.txt is really a descendant
363 of Proj1/file1.txt.
364
365 'path_offset' is the offset from source_base to the file to check ancestry for,
366 e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
367 'source_rev' is the revision ("svn log") that we're processing from the source repo.
368 'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
369 directory, when being called recursively by do_svn_add_dir().
370 'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
371 'is_dir' is whether path_offset is a directory (rather than a file).
372 """
373 ui.status(prefix + ">> do_svn_add: %s %s", source_base+"/"+path_offset+"@"+str(source_rev),
374 " (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
375 level=ui.DEBUG, color='GREEN')
376 # Check if the given path has ancestors which chain back to the current source_base
377 found_ancestor = False
378 ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+" ")
379 # ancestors[n] is the original (pre-branch-copy) trunk path.
380 # ancestors[n-1] is the first commit on the new branch.
381 copyfrom_path = ancestors[len(ancestors)-1]['path'] if ancestors else ""
382 copyfrom_rev = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
383 if ancestors:
384 # The copy-from path has ancestory back to source_url.
385 ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
386 level=ui.DEBUG, color='GREEN', bold=True)
387 found_ancestor = True
388 # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
389 # return None in the case where copyfrom_rev is *before* our source_start_rev.
390 tgt_rev = get_rev_map(copyfrom_rev, prefix+" ")
391 ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
392 else:
393 ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
394 found_ancestor = False
395 if found_ancestor and tgt_rev:
396 # Check if this path_offset in the target WC already has this ancestry, in which
397 # case there's no need to run the "svn copy" (again).
398 path_in_svn = in_svn(path_offset, prefix=prefix+" ")
399 log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+" ") else []
400 if (not log_entry or (log_entry['revision'] != tgt_rev)):
401 copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
402 ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
403 ui.status(prefix + " copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
404 ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
405 if path_in_svn and \
406 ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
407 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
408 # When being called recursively, if this child entry has the same ancestor as the
409 # the parent, then no need to try to run another "svn copy".
410 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
411 parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
412 pass
413 else:
414 # Copy this path from the equivalent path+rev in the target repo, to create the
415 # equivalent history.
416 if parent_copyfrom_path:
417 # If we have a parent copy-from path, we mis-match that so display a status
418 # message describing the action we're mimic'ing. If path_in_svn, then this
419 # is logically a "replace" rather than an "add".
420 ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
421 if path_in_svn:
422 # If local file is already under version-control, then this is a replace.
423 ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
424 run_svn(["remove", "--force", path_offset])
425 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
426 if is_dir:
427 # Export the final verison of all files in this folder.
428 add_path(export_paths, path_offset)
429 else:
430 # Export the final verison of this file.
431 run_svn(["export", "--force", "-r", source_rev,
432 source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
433 if options.keep_prop:
434 sync_svn_props(source_url, source_rev, path_offset)
435 else:
436 ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
437 else:
438 # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
439 # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
440 # Create (parent) directory if needed.
441 # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
442 # split-out to a shared tag?
443 p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
444 if not os.path.exists(p_path):
445 run_svn(["mkdir", p_path])
446 if not in_svn(path_offset, prefix=prefix+" "):
447 if is_dir:
448 # Export the final verison of all files in this folder.
449 add_path(export_paths, path_offset)
450 else:
451 # Export the final verison of this file. We *need* to do this before running
452 # the "svn add", even if we end-up re-exporting this file again via export_paths.
453 run_svn(["export", "--force", "-r", source_rev,
454 source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
455 # If not already under version-control, then "svn add" this file/folder.
456 run_svn(["add", "--parents", path_offset])
457 if options.keep_prop:
458 sync_svn_props(source_url, source_rev, path_offset)
459 if is_dir:
460 # For any folders that we process, process any child contents, so that we correctly
461 # replay copies/replaces/etc.
462 do_svn_add_dir(path_offset, source_rev, copyfrom_path, copyfrom_rev, export_paths, skip_paths, prefix+" ")
463
464 def do_svn_add_dir(path_offset, source_rev, parent_copyfrom_path, parent_copyfrom_rev, \
465 export_paths, skip_paths, prefix=""):
466 # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
467 # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
468 # associated remote repo folder. (Is this a problem?)
469 paths_local = get_svn_dirlist(path_offset)
470 paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
471 ui.status(prefix + ">> do_svn_add_dir: paths_local: %s", str(paths_local), level=ui.DEBUG, color='GREEN')
472 ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
473 # Update files/folders which exist in remote but not local
474 for path in paths_remote:
475 path_is_dir = True if path[-1] == "/" else False
476 working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
477 if not working_path in skip_paths:
478 do_svn_add(working_path, source_rev, parent_copyfrom_path, parent_copyfrom_rev,
479 export_paths, path_is_dir, skip_paths, prefix+" ")
480 # Remove files/folders which exist in local but not remote
481 for path in paths_local:
482 if not path in paths_remote:
483 ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
484 run_svn(["remove", "--force", path_offset+"/"+path])
485 # TODO: Does this handle deleted folders too? Wouldn't want to have a case
486 # where we only delete all files from folder but leave orphaned folder around.
487
488 def process_svn_log_entry(log_entry, commit_paths, prefix = ""):
489 """
490 Process SVN changes from the given log entry. Build an array (commit_paths)
491 of the paths in the working-copy that were changed, i.e. the paths which
492 we'll pass to "svn commit".
493 """
494 export_paths = []
495 source_rev = log_entry['revision']
496 ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
497 for d in log_entry['changed_paths']:
498 # Get the full path for this changed_path
499 # e.g. '/branches/bug123/projectA/file1.txt'
500 path = d['path']
501 if not is_child_path(path, source_base):
502 # Ignore changed files that are not part of this subdir
503 ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
504 continue
505 if d['kind'] == "":
506 # The "kind" value was introduced in SVN 1.6, and "svn log --xml" won't return a "kind"
507 # value for commits made on a pre-1.6 repo, even if the server is now running 1.6.
508 # We need to use other methods to fetch the node-kind for these cases.
509 d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
510 assert (d['kind'] == 'file') or (d['kind'] == 'dir')
511 path_is_dir = True if d['kind'] == 'dir' else False
512 path_is_file = True if d['kind'] == 'file' else False
513 # Calculate the offset (based on source_base) for this changed_path
514 # e.g. 'projectA/file1.txt'
515 # (path = source_base + "/" + path_offset)
516 path_offset = path[len(source_base):].strip("/")
517 # Get the action for this path
518 action = d['action']
519 if action not in _valid_svn_actions:
520 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
521 % (source_rev, action))
522 ui.status(" %s %s%s", action, d['path'],
523 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
524 level=ui.VERBOSE)
525
526 # Try to be efficient and keep track of an explicit list of paths in the
527 # working copy that changed. If we commit from the root of the working copy,
528 # then SVN needs to crawl the entire working copy looking for pending changes.
529 commit_paths.append(path_offset)
530
531 # Special-handling for replace's
532 if action == 'R':
533 # If file was "replaced" (deleted then re-added, all in same revision),
534 # then we need to run the "svn rm" first, then change action='A'. This
535 # lets the normal code below handle re-"svn add"'ing the files. This
536 # should replicate the "replace".
537 if in_svn(path_offset):
538 # Target path might not be under version-control yet, e.g. parent "add"
539 # was a copy-from a branch which had no ancestry back to trunk, and each
540 # child folder under that parent folder is a "replace" action on the final
541 # merge to trunk. Since the child folders will be in skip_paths, do_svn_add
542 # wouldn't have created them while processing the parent "add" path.
543 run_svn(["remove", "--force", path_offset])
544 action = 'A'
545
546 # Handle all the various action-types
547 # (Handle "add" first, for "svn copy/move" support)
548 if action == 'A':
549 # Determine where to export from.
550 svn_copy = False
551 # Handle cases where this "add" was a copy from another URL in the source repo
552 if d['copyfrom_revision']:
553 copyfrom_path = d['copyfrom_path']
554 copyfrom_rev = d['copyfrom_revision']
555 skip_paths = []
556 for tmp_d in log_entry['changed_paths']:
557 tmp_path = tmp_d['path']
558 if is_child_path(tmp_path, path):
559 # Build list of child entries which are also in the changed_paths list,
560 # so that do_svn_add() can skip processing these entries when recursing
561 # since we'll end-up processing them later.
562 tmp_path_offset = tmp_path[len(source_base):].strip("/")
563 skip_paths.append(tmp_path_offset)
564 do_svn_add(path_offset, source_rev, "", "", export_paths, path_is_dir, skip_paths, prefix+" ")
565 # Else just "svn export" the files from the source repo and "svn add" them.
566 else:
567 # Create (parent) directory if needed
568 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
569 if not os.path.exists(p_path):
570 run_svn(["mkdir", p_path])
571 # Export the entire added tree.
572 if path_is_dir:
573 # For directories, defer the (recurisve) "svn export". Might have a
574 # situation in a branch merge where the entry in the svn-log is a
575 # non-copy-from'd "add" but there are child contents (that we haven't
576 # gotten to yet in log_entry) that are copy-from's. When we try do
577 # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
578 # having pre-existing (svn-add'd) contents creates some trouble.
579 # Instead, just create the stub folders ("svn mkdir" above) and defer
580 # exporting the final file-state until the end.
581 add_path(export_paths, path_offset)
582 else:
583 # Export the final verison of this file. We *need* to do this before running
584 # the "svn add", even if we end-up re-exporting this file again via export_paths.
585 run_svn(["export", "--force", "-r", source_rev,
586 source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
587 if not in_svn(path_offset, prefix=prefix+" "):
588 # Need to use in_svn here to handle cases where client committed the parent
589 # folder and each indiv sub-folder.
590 run_svn(["add", "--parents", path_offset])
591 if options.keep_prop:
592 sync_svn_props(source_url, source_rev, path_offset)
593
594 elif action == 'D':
595 run_svn(["remove", "--force", path_offset])
596
597 elif action == 'M':
598 if path_is_file:
599 run_svn(["export", "--force", "-N" , "-r", source_rev,
600 source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
601 if options.keep_prop:
602 sync_svn_props(source_url, source_rev, path_offset)
603
604 else:
605 raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
606 % action)
607
608 # Export the final version of all add'd paths from source_url
609 if export_paths:
610 for path_offset in export_paths:
611 run_svn(["export", "--force", "-r", source_rev,
612 source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
613
614 def disp_svn_log_summary(log_entry):
615 ui.status("------------------------------------------------------------------------")
616 ui.status("r%s | %s | %s",
617 log_entry['revision'],
618 log_entry['author'],
619 str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
620 ui.status(log_entry['message'])
621
622 def real_main(args, parser):
623 global source_url, target_url, rev_map
624 source_url = args.pop(0).rstrip("/") # e.g. 'http://server/svn/source/trunk'
625 target_url = args.pop(0).rstrip("/") # e.g. 'file:///svn/target/trunk'
626 ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
627
628 # Make sure that both the source and target URL's are valid
629 source_info = svnclient.get_svn_info(source_url)
630 assert is_child_path(source_url, source_info['repos_url'])
631 target_info = svnclient.get_svn_info(target_url)
632 assert is_child_path(target_url, target_info['repos_url'])
633
634 # Init global vars
635 global source_repos_url,source_base,source_repos_uuid
636 source_repos_url = source_info['repos_url'] # e.g. 'http://server/svn/source'
637 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
638 source_repos_uuid = source_info['repos_uuid']
639
640 # Init start and end revision
641 try:
642 source_start_rev = svnclient.get_svn_rev(source_repos_url, options.rev_start if options.rev_start else 1)
643 except ExternalCommandFailed:
644 parser.error("invalid start source revision value: %s" % (options.rev_start))
645 try:
646 source_end_rev = svnclient.get_svn_rev(source_repos_url, options.rev_end if options.rev_end else "HEAD")
647 except ExternalCommandFailed:
648 parser.error("invalid end source revision value: %s" % (options.rev_end))
649 ui.status("Using source revision range %s:%s", source_start_rev, source_end_rev, level=ui.VERBOSE)
650
651 # TODO: If options.keep_date, should we try doing a "svn propset" on an *existing* revision
652 # as a sanity check, so we check if the pre-revprop-change hook script is correctly setup
653 # before doing first replay-commit?
654
655 target_end_rev = target_info['revision'] # Last revision # in the target repo
656 wc_target = os.path.abspath('_wc_target')
657 num_entries_proc = 0
658 commit_count = 0
659 source_rev = None
660 target_rev = None
661
662 # Check out a working copy of target_url if needed
663 wc_exists = os.path.exists(wc_target)
664 if wc_exists and not options.cont_from_break:
665 shutil.rmtree(wc_target)
666 wc_exists = False
667 if not wc_exists:
668 ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
669 svnclient.svn_checkout(target_url, wc_target)
670 os.chdir(wc_target)
671
672 if not options.cont_from_break:
673 # TODO: Warn user if trying to start (non-continue) into a non-empty target path?
674 # Get the first log entry at/after source_start_rev, which is where
675 # we'll do the initial import from.
676 it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False)
677 for source_start_log in it_log_start:
678 break
679 if not source_start_log:
680 raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
681 (source_start_rev, source_end_rev, source_url))
682
683 # This is the revision we will start from for source_url
684 source_start_rev = source_rev = int(source_start_log['revision'])
685 ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
686 ui.status("")
687
688 # For the initial commit to the target URL, export all the contents from
689 # the source URL at the start-revision.
690 disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev))
691 ui.status("(Initial import)", level=ui.VERBOSE)
692 # Export and add file-contents from source_url@source_start_rev
693 top_paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)])
694 top_paths = top_paths.strip("\n").split("\n")
695 for path in top_paths:
696 # For each top-level file/folder...
697 if not path:
698 continue
699 # Directories have a trailing slash in the "svn list" output
700 path_is_dir = True if path[-1] == "/" else False
701 path_offset = path.rstrip('/') if path_is_dir else path
702 if in_svn(path_offset, prefix=" "):
703 raise InternalError("Cannot replay history on top of pre-existing structure: %s" % source_url+"/"+path_offset)
704 if path_is_dir and not os.path.exists(path_offset):
705 os.makedirs(path_offset)
706 run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
707 run_svn(["add", path_offset])
708 # Update any properties on the newly added content
709 paths = run_svn(["list", "--recursive", "-r", source_rev, source_url+"@"+str(source_rev)])
710 paths = paths.strip("\n").split("\n")
711 if options.keep_prop:
712 sync_svn_props(source_url, source_rev, "")
713 for path in paths:
714 if not path:
715 continue
716 # Directories have a trailing slash in the "svn list" output
717 path_is_dir = True if path[-1] == "/" else False
718 path_offset = path.rstrip('/') if path_is_dir else path
719 ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path_offset, level=ui.VERBOSE)
720 if options.keep_prop:
721 sync_svn_props(source_url, source_rev, path_offset)
722 # Commit the initial import
723 num_entries_proc += 1
724 target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's
725 target_rev = commit_from_svn_log_entry(source_start_log, target_revprops=target_revprops)
726 if target_rev:
727 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
728 set_rev_map(source_rev, target_rev)
729 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
730 run_svn(["update"])
731 commit_count += 1
732 else:
733 # Re-build the rev_map based on any already-replayed history in target_url
734 build_rev_map(target_url, target_end_rev, source_info)
735 if not rev_map:
736 parser.error("called with continue-mode, but no already-replayed source history found in target_url")
737 source_start_rev = int(max(rev_map, key=rev_map.get))
738 assert source_start_rev
739 ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
740 ui.status("")
741
742 svn_vers_t = svnclient.get_svn_client_version()
743 svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
744
745 # Load SVN log starting from source_start_rev + 1
746 it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True) if source_start_rev < source_end_rev else []
747 source_rev = None
748
749 # TODO: Now that commit_from_svn_log_entry() might try to do a "svn propset svn:date",
750 # we might want some better KeyboardInterupt handilng here, to ensure that
751 # commit_from_svn_log_entry() always runs as an atomic unit.
752 try:
753 for log_entry in it_log_entries:
754 if options.entries_proc_limit:
755 if num_entries_proc >= options.entries_proc_limit:
756 break
757 # Replay this revision from source_url into target_url
758 disp_svn_log_summary(log_entry)
759 source_rev = log_entry['revision']
760 # Process all the changed-paths in this log entry
761 commit_paths = []
762 process_svn_log_entry(log_entry, commit_paths)
763 num_entries_proc += 1
764 # Commit any changes made to _wc_target
765 target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's
766 target_rev = commit_from_svn_log_entry(log_entry, commit_paths, target_revprops=target_revprops)
767 if target_rev:
768 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
769 source_rev = log_entry['revision']
770 set_rev_map(source_rev, target_rev)
771 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
772 run_svn(["update"])
773 commit_count += 1
774 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
775 if svn_vers >= 1.7 and (commit_count % 100 == 0):
776 run_svn(["cleanup"])
777 if not source_rev:
778 # If there were no new source_url revisions to process, init source_rev
779 # for the "finally" message below.
780 source_rev = source_end_rev
781
782 except KeyboardInterrupt:
783 print "\nStopped by user."
784 print "\nCleaning-up..."
785 run_svn(["cleanup"])
786 full_svn_revert()
787 except:
788 print "\nCommand failed with following error:\n"
789 traceback.print_exc()
790 print "\nCleaning-up..."
791 run_svn(["cleanup"])
792 print run_svn(["status"])
793 full_svn_revert()
794 finally:
795 print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
796
797 def main():
798 # Defined as entry point. Must be callable without arguments.
799 usage = "svn2svn, version %s\n" % str(full_version) + \
800 "<http://nynim.org/projects/svn2svn> <https://github.com/tonyduckles/svn2svn>\n\n" + \
801 "Usage: %prog [OPTIONS] source_url target_url\n"
802 description = """\
803 Replicate (replay) history from one SVN repository to another. Maintain
804 logical ancestry wherever possible, so that 'svn log' on the replayed repo
805 will correctly follow file/folder renames.
806
807 Examples:
808 Create a copy of only /trunk from source repo, starting at r5000
809 $ svnadmin create /svn/target
810 $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
811 $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
812 1. The target_url will be checked-out to ./_wc_target
813 2. The first commit to http://server/source/trunk at/after r5000 will be
814 exported & added into _wc_target
815 3. All revisions affecting http://server/source/trunk (starting at r5000)
816 will be replayed to _wc_target. Any add/copy/move/replaces that are
817 copy-from'd some path outside of /trunk (e.g. files renamed on a
818 /branch and branch was merged into /trunk) will correctly maintain
819 logical ancestry where possible.
820
821 Use continue-mode (-c) to pick-up where the last run left-off
822 $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
823 1. The target_url will be checked-out to ./_wc_target, if not already
824 checked-out
825 2. All new revisions affecting http://server/source/trunk starting from
826 the last replayed revision to file:///svn/target/trunk (based on the
827 svn2svn:* revprops) will be replayed to _wc_target, maintaining all
828 logical ancestry where possible."""
829 parser = optparse.OptionParser(usage, description=description,
830 formatter=HelpFormatter(), version="%prog "+str(full_version))
831 parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
832 help="enable additional output (use -vv or -vvv for more)")
833 parser.add_option("-a", "--archive", action="store_true", dest="archive", default=False,
834 help="archive/mirror mode; same as -UDP (see REQUIRE's below)\n"
835 "maintain same commit author, same commit time, and file/dir properties")
836 parser.add_option("-U", "--keep-author", action="store_true", dest="keep_author", default=False,
837 help="maintain same commit authors (svn:author) as source\n"
838 "(REQUIRES target_url be non-auth'd, e.g. file://-based, since this uses --username to set author)")
839 parser.add_option("-D", "--keep-date", action="store_true", dest="keep_date", default=False,
840 help="maintain same commit time (svn:date) as source\n"
841 "(REQUIRES 'pre-revprop-change' hook script to allow 'svn:date' changes)")
842 parser.add_option("-P", "--keep-prop", action="store_true", dest="keep_prop", default=False,
843 help="maintain same file/dir SVN properties as source")
844 parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
845 help="continue from last source commit to target (based on svn2svn:* revprops)")
846 parser.add_option("-r", "--revision", type="string", dest="revision", metavar="ARG",
847 help="revision range to replay from source_url\n"
848 "A revision argument can be one of:\n"
849 " START start rev # (end will be 'HEAD')\n"
850 " START:END start and ending rev #'s\n"
851 "Any revision # formats which SVN understands are "
852 "supported, e.g. 'HEAD', '{2010-01-31}', etc.")
853 parser.add_option("-u", "--log-author", action="store_true", dest="log_author", default=False,
854 help="append source commit author to replayed commit mesages")
855 parser.add_option("-d", "--log-date", action="store_true", dest="log_date", default=False,
856 help="append source commit time to replayed commit messages")
857 parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
858 help="maximum number of source revisions to process")
859 parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
860 help="process next source revision but don't commit changes to "
861 "target working-copy (forces --limit=1)")
862 parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
863 help="enable debugging output (same as -vvv)")
864 global options
865 options, args = parser.parse_args()
866 if len(args) != 2:
867 parser.error("incorrect number of arguments")
868 if options.verbosity < 10:
869 # Expand multiple "-v" arguments to a real ui._level value
870 options.verbosity *= 10
871 if options.dry_run:
872 # When in dry-run mode, only try to process the next log_entry
873 options.entries_proc_limit = 1
874 options.rev_start = None
875 options.rev_end = None
876 if options.revision:
877 # Reg-ex for matching a revision arg (http://svnbook.red-bean.com/en/1.5/svn.tour.revs.specifiers.html#svn.tour.revs.dates)
878 rev_patt = '[0-9A-Z]+|\{[0-9A-Za-z/\\ :-]+\}'
879 rev = None
880 match = re.match('^('+rev_patt+'):('+rev_patt+')$', options.revision) # First try start:end match
881 if match is None: match = re.match('^('+rev_patt+')$', options.revision) # Next, try start match
882 if match is None:
883 parser.error("unexpected --revision argument format; see 'svn help log' for valid revision formats")
884 rev = match.groups()
885 options.rev_start = rev[0] if len(rev)>0 else None
886 options.rev_end = rev[1] if len(rev)>1 else None
887 if options.archive:
888 options.keep_author = True
889 options.keep_date = True
890 options.keep_prop = True
891 ui.update_config(options)
892 return real_main(args, parser)
893
894
895 if __name__ == "__main__":
896 sys.exit(main() or 0)