]> Tony Duckles's Git Repositories (git.nynim.org) - svn2svn.git/blob - svn2svn/run/svn2svn.py
Add optional 'no_fail' param to run_svn() and downstream functions
[svn2svn.git] / svn2svn / run / svn2svn.py
1 """
2 Replicate (replay) changesets from one SVN repository to another:
3 * Maintains full logical history (e.g. uses "svn copy" for renames).
4 * Maintains original commit messages.
5 * Optionally maintain source author info. (Only supported if accessing
6 target SVN repo via file://)
7 * Cannot maintain original commit date, but appends original commit date
8 for each commit message: "Date: %d".
9 * Optionally run an external shell script before each replayed commit
10 to give the ability to dynamically exclude or modify files as part
11 of the replay.
12
13 License: GPLv3, same as hgsvn (https://bitbucket.org/andialbrecht/hgsvn)
14 Author: Tony Duckles (https://github.com/tonyduckles/svn2svn)
15 (Inspired by http://code.google.com/p/svn2svn/, and uses code for hgsvn
16 for SVN client handling)
17 """
18
19 from .. import base_version, full_version
20 from .. import ui
21 from .. import svnclient
22 from ..shell import run_svn
23 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction)
24
25 import sys
26 import os
27 import time
28 import traceback
29 from optparse import OptionParser,OptionGroup
30 from datetime import datetime
31 from operator import itemgetter
32
33 def commit_from_svn_log_entry(entry, files=None, keep_author=False, revprops=[]):
34 """
35 Given an SVN log entry and an optional sequence of files, do an svn commit.
36 """
37 # TODO: Run optional external shell hook here, for doing pre-commit filtering
38 # This will use the local timezone for displaying commit times
39 timestamp = int(entry['date'])
40 svn_date = str(datetime.fromtimestamp(timestamp))
41 # Uncomment this one one if you prefer UTC commit times
42 #svn_date = "%d 0" % timestamp
43 if keep_author:
44 options = ["ci", "--force-log", "-m", entry['message'] + "\nDate: " + svn_date, "--username", entry['author']]
45 else:
46 options = ["ci", "--force-log", "-m", entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + entry['author']]
47 if revprops:
48 for r in revprops:
49 options += ["--with-revprop", r['name']+"="+str(r['value'])]
50 if files:
51 options += list(files)
52 print "(Committing source rev #"+str(entry['revision'])+"...)"
53 run_svn(options)
54
55 def in_svn(p, in_repo=False):
56 """
57 Check if a given file/folder is being tracked by Subversion.
58 Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
59 With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
60 Use "svn status" to check the status of the file/folder.
61 """
62 entries = svnclient.get_svn_status(p)
63 if not entries:
64 return False
65 d = entries[0]
66 # If caller requires this path to be in the SVN repo, prevent returning True for locally-added paths.
67 if in_repo and (d['status'] == 'added' or d['revision'] is None):
68 return False
69 return True if (d['type'] == 'normal' or d['status'] == 'added') else False
70
71 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
72 """
73 Given a source path, walk the SVN history backwards to inspect the ancestory of
74 that path, seeing if it traces back to base_path. Build an array of copyfrom_path
75 and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
76 which base_path is a substring match of (e.g. we crawled back to the initial branch-
77 copy from trunk), then return the collection of ancestor paths. Otherwise,
78 copyfrom_path has no ancestory compared to base_path.
79
80 This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
81 file/folder was renamed in a branch and then that branch was merged back to trunk.
82
83 'svn_repos_url' is the full URL to the root of the SVN repository,
84 e.g. 'file:///path/to/repo'
85 'base_path' is the path in the SVN repo to the target path we're trying to
86 trace ancestry back to, e.g. 'trunk'.
87 'source_path' is the path in the SVN repo to the source path to start checking
88 ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
89 (full_path = svn_repos_url+base_path+"/"+path_offset)
90 'source_rev' is the revision to start walking the history of source_path backwards from.
91 """
92 if debug:
93 print prefix+"\x1b[33m" + ">> find_svn_ancestors: Start: ("+svn_repos_url+") source_path: "+source_path+"@"+str(source_rev)+" base_path: "+base_path + "\x1b[0m"
94 done = False
95 working_path = base_path+"/"+source_path
96 working_rev = source_rev
97 first_iter_done = False
98 ancestors_temp = []
99 while not done:
100 # Get the first "svn log" entry for this path (relative to @rev)
101 if debug:
102 print prefix+"\x1b[33m" + ">> find_svn_ancestors: " + svn_repos_url + working_path+"@"+str(working_rev) + "\x1b[0m"
103 log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path+"@"+str(working_rev), 1, str(working_rev), True)
104 if not log_entry:
105 if debug:
106 print prefix+"\x1b[33m" + ">> find_svn_ancestors: Done: no log_entry" + "\x1b[0m"
107 done = True
108 break
109 # If we found a copy-from case which matches our base_path, we're done.
110 # ...but only if we've at least tried to search for the first copy-from path.
111 if first_iter_done and working_path.startswith(base_path):
112 if debug:
113 print prefix+"\x1b[33m" + ">> find_svn_ancestors: Done: Found working_path.startswith(base_path) and first_iter_done=True" + "\x1b[0m"
114 done = True
115 break
116 first_iter_done = True
117 # Search for any actions on our target path (or parent paths).
118 changed_paths_temp = []
119 for d in log_entry['changed_paths']:
120 path = d['path']
121 if path in working_path:
122 changed_paths_temp.append({'path': path, 'data': d})
123 if not changed_paths_temp:
124 # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
125 if debug:
126 print prefix+"\x1b[33m" + ">> find_svn_ancestors: Done: No matching changed_paths" + "\x1b[0m"
127 done = True
128 continue
129 # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
130 changed_paths = sorted(changed_paths_temp, key=itemgetter('path'), reverse=True)
131 # Find the action for our working_path in this revision. Use a loop to check in reverse order,
132 # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
133 for v in changed_paths:
134 d = v['data']
135 path = d['path']
136 # Check action-type for this file
137 action = d['action']
138 if action not in 'MARD':
139 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
140 % (log_entry['revision'], action))
141 if debug:
142 debug_desc = "> " + action + " " + path
143 if d['copyfrom_path']:
144 debug_desc += " (from " + d['copyfrom_path']+"@"+str(d['copyfrom_revision']) + ")"
145 print prefix+"\x1b[33m" + debug_desc + "\x1b[0m"
146 if action == 'D':
147 # If file/folder was deleted, it has no ancestor
148 ancestors_temp = []
149 if debug:
150 print prefix+"\x1b[33m" + ">> find_svn_ancestors: Done: deleted" + "\x1b[0m"
151 done = True
152 break
153 if action in 'RA':
154 # If file/folder was added/replaced but not a copy, it has no ancestor
155 if not d['copyfrom_path']:
156 ancestors_temp = []
157 if debug:
158 print prefix+"\x1b[33m" + ">> find_svn_ancestors: Done: "+("Added" if action == "A" else "Replaced")+" with no copyfrom_path" + "\x1b[0m"
159 done = True
160 break
161 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
162 # and keep checking for ancestors
163 if debug:
164 print prefix+"\x1b[33m" + ">> find_svn_ancestors: Found copy-from ("+action+"): " + \
165 path + " --> " + d['copyfrom_path']+"@"+str(d['copyfrom_revision']) + "\x1b[0m"
166 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
167 'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
168 working_path = working_path.replace(d['path'], d['copyfrom_path'])
169 working_rev = d['copyfrom_revision']
170 # Follow the copy and keep on searching
171 break
172 ancestors = []
173 if ancestors_temp:
174 ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
175 working_path = base_path+"/"+source_path
176 for idx in range(len(ancestors_temp)):
177 d = ancestors_temp[idx]
178 working_path = working_path.replace(d['path'], d['copyfrom_path'])
179 working_rev = d['copyfrom_rev']
180 ancestors.append({'path': working_path, 'revision': working_rev})
181 if debug:
182 max_len = 0
183 for idx in range(len(ancestors)):
184 d = ancestors[idx]
185 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
186 print prefix+"\x1b[93m" + ">> find_svn_ancestors: Found parent ancestors: " + "\x1b[0m"
187 for idx in range(len(ancestors)-1):
188 d = ancestors[idx]
189 d_next = ancestors[idx+1]
190 print prefix+"\x1b[33m" + " ["+str(idx)+"] " + str(d['path']+"@"+str(d['revision'])).ljust(max_len) + \
191 " <-- " + str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len) + "\x1b[0m"
192 else:
193 if debug:
194 print prefix+"\x1b[33m" + ">> find_svn_ancestors: No ancestor-chain found: " + svn_repos_url+base_path+"/"+source_path+"@"+(str(source_rev)) + "\x1b[0m"
195 return ancestors
196
197 def get_rev_map(rev_map, src_rev, prefix):
198 """
199 Find the equivalent rev # in the target repo for the given rev # from the source repo.
200 """
201 if debug:
202 print prefix + "\x1b[32m" + ">> get_rev_map("+str(src_rev)+")" + "\x1b[0m"
203 # Find the highest entry less-than-or-equal-to src_rev
204 for rev in range(src_rev, 0, -1):
205 if debug:
206 print prefix + "\x1b[32m" + ">> get_rev_map: rev="+str(rev)+" in_rev_map="+str(rev in rev_map) + "\x1b[0m"
207 if rev in rev_map:
208 return rev_map[rev]
209 # Else, we fell off the bottom of the rev_map. Ruh-roh...
210 return None
211
212 def get_svn_dirlist(svn_path, svn_rev = ""):
213 """
214 Get a list of all the child contents (recusive) of the given folder path.
215 """
216 args = ["list"]
217 path = svn_path
218 if svn_rev:
219 args += ["-r", str(svn_rev)]
220 path += "@"+str(svn_rev)
221 args += [path]
222 paths = run_svn(args, False, True)
223 paths = paths.strip("\n").split("\n") if len(paths)>1 else []
224 return paths
225
226 def _add_export_path(export_paths, path_offset):
227 found = False
228 for p in export_paths:
229 if path_offset.startswith(p):
230 found = True
231 break
232 if not found:
233 export_paths.append(path_offset)
234 return export_paths
235
236 def do_svn_add(source_repos_url, source_url, path_offset, target_url, source_rev, \
237 parent_copyfrom_path="", parent_copyfrom_rev="", export_paths={}, \
238 rev_map={}, is_dir = False, prefix = ""):
239 """
240 Given the add'd source path, replay the "svn add/copy" commands to correctly
241 track renames across copy-from's.
242
243 For example, consider a sequence of events like this:
244 1. svn copy /trunk /branches/fix1
245 2. (Make some changes on /branches/fix1)
246 3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2 " Rename folder
247 4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt " Rename file inside renamed folder
248 5. svn co /trunk && svn merge /branches/fix1
249 After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
250 and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
251 to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
252 we'd lose the logical history that Proj2/file2.txt is really a descendant
253 of Proj1/file1.txt.
254
255 'source_repos_url' is the full URL to the root of the source repository.
256 'source_url' is the full URL to the source path in the source repository.
257 'path_offset' is the offset from source_base to the file to check ancestry for,
258 e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
259 'target_url' is the full URL to the target path in the target repository.
260 'source_rev' is the revision ("svn log") that we're processing from the source repo.
261 'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
262 directory, when being called recursively by do_svn_add_dir().
263 'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
264 'rev_map' is the running mapping-table dictionary for source-repo rev #'s
265 to the equivalent target-repo rev #'s.
266 'is_dir' is whether path_offset is a directory (rather than a file).
267 """
268 source_base = source_url[len(source_repos_url):]
269 if debug:
270 print prefix + "\x1b[32m" + ">> do_svn_add: " + source_base+"/"+path_offset+"@"+str(source_rev) + \
271 (" (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "") + "\x1b[0m"
272 # Check if the given path has ancestors which chain back to the current source_base
273 found_ancestor = False
274 ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+" ")
275 # ancestors[n] is the original (pre-branch-copy) trunk path.
276 # ancestors[n-1] is the first commit on the new branch.
277 copyfrom_path = ancestors[len(ancestors)-1]['path'] if ancestors else ""
278 copyfrom_rev = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
279 if ancestors:
280 # The copy-from path has ancestory back to source_url.
281 if debug:
282 print prefix + "\x1b[32;1m" + ">> do_svn_add: Check copy-from: Found parent: " + copyfrom_path+"@"+str(copyfrom_rev) + "\x1b[0m"
283 found_ancestor = True
284 # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
285 # return None in the case where copyfrom_rev is *before* our source_start_rev.
286 tgt_rev = get_rev_map(rev_map, copyfrom_rev, prefix+" ")
287 if debug:
288 print prefix + "\x1b[32m" + ">> do_svn_add: get_rev_map: " + str(copyfrom_rev) + " (source) -> " + str(tgt_rev) + " (target)" + "\x1b[0m"
289 else:
290 if debug:
291 print prefix + "\x1b[32;1m" + ">> do_svn_add: Check copy-from: No ancestor chain found." + "\x1b[0m"
292 found_ancestor = False
293 if found_ancestor and tgt_rev:
294 # Check if this path_offset in the target WC already has this ancestry, in which
295 # case there's no need to run the "svn copy" (again).
296 path_in_svn = in_svn(path_offset)
297 log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, True) else []
298 if (not log_entry or (log_entry['revision'] != tgt_rev)):
299 copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
300 if debug:
301 print prefix + "\x1b[32m" + ">> do_svn_add: svn_copy: Copy-from: " + copyfrom_path+"@"+str(copyfrom_rev) + "\x1b[0m"
302 print prefix + "in_svn("+path_offset+") = " + str(path_in_svn)
303 print prefix + "copyfrom_path: "+copyfrom_path+" parent_copyfrom_path: "+parent_copyfrom_path
304 print prefix + "copyfrom_rev: "+str(copyfrom_rev)+" parent_copyfrom_rev: "+str(parent_copyfrom_rev)
305 if path_in_svn and \
306 ((parent_copyfrom_path and copyfrom_path.startswith(parent_copyfrom_path)) and \
307 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
308 # When being called recursively, if this child entry has the same ancestor as the
309 # the parent, then no need to try to run another "svn copy".
310 if debug:
311 print prefix + "\x1b[32m" + ">> do_svn_add: svn_copy: Same ancestry as parent: " + parent_copyfrom_path+"@"+str(parent_copyfrom_rev) + "\x1b[0m"
312 pass
313 else:
314 # Copy this path from the equivalent path+rev in the target repo, to create the
315 # equivalent history.
316 if parent_copyfrom_path and svnlog_verbose:
317 # If we have a parent copy-from path, we mis-match that so display a status
318 # message describing the action we're mimic'ing. If path_in_svn, then this
319 # is logically a "replace" rather than an "add".
320 print " "+('R' if path_in_svn else 'A')+" "+source_base+"/"+path_offset+" (from "+ancestors[1]['path']+"@"+str(copyfrom_rev)+")"
321 if path_in_svn:
322 # If local file is already under version-control, then this is a replace.
323 if debug:
324 print prefix + "\x1b[32m" + ">> do_svn_add: pre-copy: local path already exists: " + path_offset + "\x1b[0m"
325 run_svn(["remove", "--force", path_offset])
326 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
327 # Export the final version of this file/folder from the source repo, to make
328 # sure we're up-to-date.
329 export_paths = _add_export_path(export_paths, path_offset)
330 else:
331 print prefix + "\x1b[32m" + ">> do_svn_add: Skipped 'svn copy': " + path_offset + "\x1b[0m"
332 else:
333 # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
334 # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
335 # Create (parent) directory if needed.
336 # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
337 # split-out to a shared tag?
338 p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
339 if not os.path.exists(p_path):
340 run_svn(["mkdir", p_path])
341 if not in_svn(path_offset):
342 if is_dir:
343 # Export the final verison of all files in this folder.
344 export_paths = _add_export_path(export_paths, path_offset)
345 else:
346 # Export the final verison of this file. We *need* to do this before running
347 # the "svn add", even if we end-up re-exporting this file again via export_paths.
348 run_svn(["export", "--force", "-r", str(source_rev),
349 source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
350 # If not already under version-control, then "svn add" this file/folder.
351 run_svn(["add", "--parents", path_offset])
352 # TODO: Need to copy SVN properties from source repos
353 if is_dir:
354 # For any folders that we process, process any child contents, so that we correctly
355 # replay copies/replaces/etc.
356 do_svn_add_dir(source_repos_url, source_url, path_offset, source_rev, target_url,
357 copyfrom_path, copyfrom_rev, export_paths, rev_map, prefix+" ")
358
359 def do_svn_add_dir(source_repos_url, source_url, path_offset, source_rev, target_url, \
360 parent_copyfrom_path, parent_copyfrom_rev, export_paths, rev_map, prefix=""):
361 source_base = source_url[len(source_repos_url):]
362 # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
363 # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
364 # associated remote repo folder. (Is this a problem?)
365 paths_local = get_svn_dirlist(path_offset)
366 paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
367 if debug:
368 print prefix + "\x1b[32m" + ">> do_svn_add_dir: paths_local: " + str(paths_local) + "\x1b[0m"
369 print prefix + "\x1b[32m" + ">> do_svn_add_dir: paths_remote: " + str(paths_remote) + "\x1b[0m"
370 # Update files/folders which exist in remote but not local
371 for path in paths_remote:
372 path_is_dir = True if path[-1] == "/" else False
373 working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
374 do_svn_add(source_repos_url, source_url, working_path, target_url, source_rev,
375 parent_copyfrom_path, parent_copyfrom_rev, export_paths,
376 rev_map, path_is_dir, prefix+" ")
377 # Remove files/folders which exist in local but not remote
378 for path in paths_local:
379 if not path in paths_remote:
380 if svnlog_verbose:
381 print " D " + source_base+"/"+path_offset+"/"+path
382 run_svn(["remove", "--force", path_offset+"/"+path])
383 # TODO: Does this handle deleted folders too? Wouldn't want to have a case
384 # where we only delete all files from folder but leave orphaned folder around.
385
386 def process_svn_log_entry(log_entry, source_repos_url, source_url, target_url, \
387 rev_map, commit_paths = [], prefix = ""):
388 """
389 Process SVN changes from the given log entry.
390 Returns array of all the paths in the working-copy that were changed,
391 i.e. the paths which need to be "svn commit".
392
393 'log_entry' is the array structure built by parse_svn_log_xml().
394 'source_repos_url' is the full URL to the root of the source repository.
395 'source_url' is the full URL to the source path in the source repository.
396 'target_url' is the full URL to the target path in the target repository.
397 'rev_map' is the running mapping-table dictionary for source-repo rev #'s
398 to the equivalent target-repo rev #'s.
399 'commit_paths' is the working list of specific paths which changes to pass
400 to the final "svn commit".
401 """
402 removed_paths = []
403 export_paths = []
404 # Get the relative offset of source_url based on source_repos_url
405 # e.g. '/branches/bug123'
406 source_base = source_url[len(source_repos_url):]
407 source_rev = log_entry['revision']
408 if debug:
409 print prefix + "\x1b[32m" + ">> process_svn_log_entry: " + source_url+"@"+str(source_rev) + "\x1b[0m"
410 for d in log_entry['changed_paths']:
411 # Get the full path for this changed_path
412 # e.g. '/branches/bug123/projectA/file1.txt'
413 path = d['path']
414 if not path.startswith(source_base + "/"):
415 # Ignore changed files that are not part of this subdir
416 if path != source_base:
417 if debug:
418 print prefix + "\x1b[90m" + ">> process_svn_log_entry: Unrelated path: " + path + " (" + source_base + ")" + "\x1b[0m"
419 continue
420 # Calculate the offset (based on source_base) for this changed_path
421 # e.g. 'projectA/file1.txt'
422 # (path = source_base + "/" + path_offset)
423 path_offset = path[len(source_base):].strip("/")
424 # Get the action for this path
425 action = d['action']
426 if action not in 'MARD':
427 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
428 % (source_rev, action))
429 if svnlog_verbose and (action not in 'D'):
430 # (Note: Skip displaying action message for 'D' here since we'll display that
431 # message when we process the deferred delete actions at the end.)
432 msg = " " + action + " " + d['path']
433 if d['copyfrom_path']:
434 msg += " (from " + d['copyfrom_path']+"@"+str(d['copyfrom_revision']) + ")"
435 print prefix + msg
436
437 # Try to be efficient and keep track of an explicit list of paths in the
438 # working copy that changed. If we commit from the root of the working copy,
439 # then SVN needs to crawl the entire working copy looking for pending changes.
440 # But, if we gather too many paths to commit, then we wipe commit_paths below
441 # and end-up doing a commit at the root of the working-copy.
442 if len (commit_paths) < 100:
443 commit_paths.append(path_offset)
444
445 # Special-handling for replace's
446 if action == 'R':
447 # If file was "replaced" (deleted then re-added, all in same revision),
448 # then we need to run the "svn rm" first, then change action='A'. This
449 # lets the normal code below handle re-"svn add"'ing the files. This
450 # should replicate the "replace".
451 run_svn(["remove", "--force", path_offset])
452 action = 'A'
453
454 # Handle all the various action-types
455 # (Handle "add" first, for "svn copy/move" support)
456 if action == 'A':
457 # If we have any queued deletions for this same path, remove those if we're re-adding this path.
458 if path_offset in removed_paths:
459 removed_paths.remove(path_offset)
460 # Determine where to export from.
461 svn_copy = False
462 path_is_dir = True if d['kind'] == 'dir' else False
463 # Handle cases where this "add" was a copy from another URL in the source repos
464 if d['copyfrom_revision']:
465 copyfrom_path = d['copyfrom_path']
466 copyfrom_rev = d['copyfrom_revision']
467 do_svn_add(source_repos_url, source_url, path_offset, target_url, source_rev,
468 "", "", export_paths, rev_map, path_is_dir, prefix+" ")
469 # Else just "svn export" the files from the source repo and "svn add" them.
470 else:
471 # Create (parent) directory if needed
472 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
473 if not os.path.exists(p_path):
474 run_svn(["mkdir", p_path])
475 # Export the entire added tree.
476 if path_is_dir:
477 export_paths = _add_export_path(export_paths, path_offset)
478 else:
479 # Export the final verison of this file. We *need* to do this before running
480 # the "svn add", even if we end-up re-exporting this file again via export_paths.
481 run_svn(["export", "--force", "-r", str(source_rev),
482 source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
483 # TODO: Do we need the in_svn check here?
484 #if not in_svn(path_offset):
485 run_svn(["add", "--parents", path_offset])
486 # TODO: Need to copy SVN properties from source repos
487
488 elif action == 'D':
489 # Queue "svn remove" commands, to allow the action == 'A' handling the opportunity
490 # to do smart "svn copy" handling on copy/move/renames.
491 if not path_offset in removed_paths:
492 removed_paths.append(path_offset)
493
494 elif action == 'M':
495 # TODO: Is "svn merge -c" correct here? Should this just be an "svn export" plus
496 # proplist updating?
497 out = run_svn(["merge", "-c", str(source_rev), "--non-recursive",
498 "--non-interactive", "--accept=theirs-full",
499 source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
500
501 else:
502 raise SVNError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
503 % action)
504
505 # Process any deferred removed actions
506 if removed_paths:
507 path_base = source_url[len(source_repos_url):]
508 for path_offset in removed_paths:
509 if svnlog_verbose:
510 print " D " + path_base+"/"+path_offset
511 run_svn(["remove", "--force", path_offset])
512 # Export the final version of all add'd paths from source_url
513 if export_paths:
514 for path_offset in export_paths:
515 run_svn(["export", "--force", "-r", str(source_rev),
516 source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
517
518 return commit_paths
519
520 def disp_svn_log_summary(log_entry):
521 print "\n(Starting source rev #"+str(log_entry['revision'])+":)"
522 print "r"+str(log_entry['revision']) + " | " + \
523 log_entry['author'] + " | " + \
524 str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' '))
525 print log_entry['message']
526 print "------------------------------------------------------------------------"
527
528 def pull_svn_rev(log_entry, source_repos_url, source_repos_uuid, source_url, target_url, rev_map, keep_author=False):
529 """
530 Pull SVN changes from the given log entry.
531 Returns the new SVN revision.
532 If an exception occurs, it will rollback to revision 'source_rev - 1'.
533 """
534 disp_svn_log_summary(log_entry)
535 source_rev = log_entry['revision']
536
537 # Process all the paths in this log entry
538 commit_paths = []
539 process_svn_log_entry(log_entry, source_repos_url, source_url, target_url,
540 rev_map, commit_paths)
541 # If we had too many individual paths to commit, wipe the list and just commit at
542 # the root of the working copy.
543 if len (commit_paths) > 99:
544 commit_paths = []
545
546 # Add source-tracking revprop's
547 revprops = [{'name':'source_uuid', 'value':source_repos_uuid},
548 {'name':'source_url', 'value':source_url},
549 {'name':'source_rev', 'value':source_rev}]
550 commit_from_svn_log_entry(log_entry, commit_paths, keep_author=keep_author, revprops=revprops)
551 print "(Finished source rev #"+str(source_rev)+")"
552
553 def run_parser(parser):
554 """
555 Add common options to an OptionParser instance, and run parsing.
556 """
557 parser.add_option("", "--version", dest="show_version", action="store_true",
558 help="show version and exit")
559 parser.remove_option("--help")
560 parser.add_option("-h", "--help", dest="show_help", action="store_true",
561 help="show this help message and exit")
562 parser.add_option("-v", "--verbose", dest="verbosity", const=20,
563 default=10, action="store_const",
564 help="enable additional output")
565 parser.add_option("--debug", dest="verbosity", const=30,
566 action="store_const",
567 help="enable debugging output")
568 options, args = parser.parse_args()
569 if options.show_help:
570 parser.print_help()
571 sys.exit(0)
572 if options.show_version:
573 prog_name = os.path.basename(sys.argv[0])
574 print prog_name, full_version
575 sys.exit(0)
576 ui.update_config(options)
577 return options, args
578
579 def display_parser_error(parser, message):
580 """
581 Display an options error, and terminate.
582 """
583 print "error:", message
584 print
585 parser.print_help()
586 sys.exit(1)
587
588 def real_main(options, args):
589 source_url = args.pop(0).rstrip("/")
590 target_url = args.pop(0).rstrip("/")
591 if options.keep_author:
592 keep_author = True
593 else:
594 keep_author = False
595
596 # Find the greatest_rev in the source repo
597 svn_info = svnclient.get_svn_info(source_url)
598 greatest_rev = svn_info['revision']
599 # Get the base URL for the source repos, e.g. 'svn://svn.example.com/svn/repo'
600 source_repos_url = svn_info['repos_url']
601 # Get the UUID for the source repos
602 source_repos_uuid = svn_info['repos_uuid']
603
604 wc_target = "_wc_target"
605 rev_map = {}
606
607 # if old working copy does not exist, disable continue mode
608 # TODO: Better continue support. Maybe include source repo's rev # in target commit info?
609 if not os.path.exists(wc_target):
610 options.cont_from_break = False
611
612 if not options.cont_from_break:
613 # Get log entry for the SVN revision we will check out
614 if options.svn_rev:
615 # If specify a rev, get log entry just before or at rev
616 svn_start_log = svnclient.get_last_svn_log_entry(source_url, 1, options.svn_rev, False)
617 else:
618 # Otherwise, get log entry of branch creation
619 # TODO: This call is *very* expensive on a repo with lots of revisions.
620 # Even though the call is passing --limit 1, it seems like that limit-filter
621 # is happening after SVN has fetched the full log history.
622 svn_start_log = svnclient.get_first_svn_log_entry(source_url, 1, greatest_rev, False)
623
624 # This is the revision we will start from for source_url
625 source_start_rev = svn_start_log['revision']
626
627 # Check out a working copy of target_url
628 wc_target = os.path.abspath(wc_target)
629 if os.path.exists(wc_target):
630 shutil.rmtree(wc_target)
631 svnclient.svn_checkout(target_url, wc_target)
632 os.chdir(wc_target)
633
634 # For the initial commit to the target URL, export all the contents from
635 # the source URL at the start-revision.
636 paths = run_svn(["list", "-r", str(source_start_rev), source_url+"@"+str(source_start_rev)])
637 if len(paths)>1:
638 disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_start_rev, source_start_rev))
639 print "(Initial import)"
640 paths = paths.strip("\n").split("\n")
641 for path in paths:
642 # For each top-level file/folder...
643 if not path:
644 # Skip null lines
645 break
646 # Directories have a trailing slash in the "svn list" output
647 path_is_dir = True if path[-1] == "/" else False
648 if path_is_dir:
649 path=path.rstrip('/')
650 if not os.path.exists(path):
651 os.makedirs(path)
652 run_svn(["export", "--force", "-r" , str(source_start_rev), source_url+"/"+path+"@"+str(source_start_rev), path])
653 run_svn(["add", path])
654 revprops = [{'name':'source_uuid', 'value':source_repos_uuid},
655 {'name':'source_url', 'value':source_url},
656 {'name':'source_rev', 'value':source_start_rev}]
657 commit_from_svn_log_entry(svn_start_log, [], keep_author=keep_author, revprops=revprops)
658 print "(Finished source rev #"+str(source_start_rev)+")"
659 else:
660 wc_target = os.path.abspath(wc_target)
661 os.chdir(wc_target)
662 # TODO: Need better resume support. For the time being, expect caller explictly passes in resume revision.
663 source_start_rev = options.svn_rev
664 if source_start_rev < 1:
665 display_error("Invalid arguments\n\nNeed to pass result rev # (-r) when using continue-mode (-c)", False)
666
667 # Load SVN log starting from source_start_rev + 1
668 it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev + 1, greatest_rev)
669
670 try:
671 for log_entry in it_log_entries:
672 # Replay this revision from source_url into target_url
673 pull_svn_rev(log_entry, source_repos_url, source_repos_uuid, source_url,
674 target_url, rev_map, keep_author)
675 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
676 run_svn(["up"])
677 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
678 dup_info = get_svn_info(target_url)
679 dup_rev = dup_info['revision']
680 source_rev = log_entry['revision']
681 if debug:
682 print "\x1b[32m" + ">> main: rev_map.add: source_rev=%s target_rev=%s" % (source_rev, dup_rev) + "\x1b[0m"
683 rev_map[source_rev] = dup_rev
684
685 except KeyboardInterrupt:
686 print "\nStopped by user."
687 run_svn(["cleanup"])
688 run_svn(["revert", "--recursive", "."])
689 # TODO: Run "svn status" and pro-actively delete any "?" orphaned entries, to clean-up the WC?
690 except:
691 print "\nCommand failed with following error:\n"
692 traceback.print_exc()
693 run_svn(["cleanup"])
694 run_svn(["revert", "--recursive", "."])
695 # TODO: Run "svn status" and pro-actively delete any "?" orphaned entries, to clean-up the WC?
696 finally:
697 run_svn(["up"])
698 print "\nFinished!"
699
700 def main():
701 # Defined as entry point. Must be callable without arguments.
702 usage = "Usage: %prog [OPTIONS] source_url target_url"
703 parser = OptionParser(usage)
704 parser.add_option("-r", "--revision", type="int", dest="svn_rev", metavar="REV",
705 help="initial SVN revision to checkout from")
706 parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author",
707 help="maintain original Author info from source repo")
708 parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
709 help="continue from previous break")
710 (options, args) = run_parser(parser)
711 if len(args) != 2:
712 display_parser_error(parser, "incorrect number of arguments")
713 return real_main(options, args)
714
715
716 if __name__ == "__main__":
717 sys.exit(main() or 0)