]> Tony Duckles's Git Repositories (git.nynim.org) - svn2svn.git/blob - svn2svn/run/svn2svn.py
Fix iter_svn_log_entries() to handle chunks which have no matching entries.
[svn2svn.git] / svn2svn / run / svn2svn.py
1 """
2 Replicate (replay) changesets from one SVN repository to another.
3 """
4
5 from .. import base_version, full_version
6 from .. import ui
7 from .. import svnclient
8 from ..shell import run_svn
9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
10 from parse import HelpFormatter
11
12 import sys
13 import os
14 import time
15 import traceback
16 import shutil
17 import operator
18 import optparse
19 from datetime import datetime
20
21 _valid_svn_actions = "MARD" # The list of known SVN action abbr's, from "svn log"
22
23 # Module-level variables/parameters
24 source_url = "" # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
25 source_repos_url = "" # URL to root of source SVN repo, e.g. 'http://server/svn/source'
26 source_base = "" # Relative path of source_url in source SVN repo, e.g. '/trunk'
27 source_repos_uuid = "" # UUID of source SVN repo
28 target_url ="" # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
29 rev_map = {} # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
30
31 def commit_from_svn_log_entry(log_entry, options, commit_paths=None, target_revprops=None):
32 """
33 Given an SVN log entry and an optional list of changed paths, do an svn commit.
34 """
35 # TODO: Run optional external shell hook here, for doing pre-commit filtering
36 # Display the _wc_target "svn status" info if running in -vv (or higher) mode
37 if ui.get_level() >= ui.EXTRA:
38 ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
39 ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
40 # This will use the local timezone for displaying commit times
41 timestamp = int(log_entry['date'])
42 svn_date = str(datetime.fromtimestamp(timestamp))
43 # Uncomment this one one if you prefer UTC commit times
44 #svn_date = "%d 0" % timestamp
45 args = ["commit", "--force-log"]
46 if options.keep_author:
47 args += ["-m", log_entry['message'] + "\nDate: " + svn_date, "--username", log_entry['author']]
48 else:
49 args += ["-m", log_entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + log_entry['author']]
50 revprops = {}
51 if log_entry['revprops']:
52 # Carry forward any revprop's from the source revision
53 for v in log_entry['revprops']:
54 revprops[v['name']] = v['value']
55 if target_revprops:
56 # Add any extra revprop's we want to set for the target repo commits
57 for v in target_revprops:
58 revprops[v['name']] = v['value']
59 if revprops:
60 for key in revprops:
61 args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
62 if commit_paths:
63 if len(commit_paths)<100:
64 # If we don't have an excessive amount of individual changed paths, pass
65 # those to the "svn commit" command. Else, pass nothing so we commit at
66 # the root of the working-copy.
67 args += list(commit_paths)
68 rev = None
69 if not options.dry_run:
70 # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
71 output = run_svn(args)
72 if output:
73 output_lines = output.strip("\n").split("\n")
74 rev = ""
75 for line in output_lines:
76 if line[0:19] == 'Committed revision ':
77 rev = line[19:].rstrip('.')
78 break
79 if rev:
80 ui.status("Committed revision %s.", rev)
81 return rev
82
83 def full_svn_revert():
84 """
85 Do an "svn revert" and proactively remove any extra files in the working copy.
86 """
87 run_svn(["revert", "--recursive", "."])
88 output = run_svn(["status"])
89 if output:
90 output_lines = output.strip("\n").split("\n")
91 for line in output_lines:
92 if line[0] == "?":
93 path = line[4:].strip(" ")
94 if os.path.isfile(path):
95 os.remove(path)
96 if os.path.isdir(path):
97 shutil.rmtree(path)
98
99 def gen_tracking_revprops(source_rev):
100 """
101 Build an array of svn2svn-specific source-tracking revprops.
102 """
103 revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
104 {'name':'svn2svn:source_url', 'value':source_url},
105 {'name':'svn2svn:source_rev', 'value':source_rev}]
106 return revprops
107
108 def in_svn(p, require_in_repo=False, prefix=""):
109 """
110 Check if a given file/folder is being tracked by Subversion.
111 Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
112 With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
113 Use "svn status" to check the status of the file/folder.
114 """
115 entries = svnclient.get_svn_status(p, no_recursive=True)
116 if not entries:
117 return False
118 d = entries[0]
119 if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
120 # If caller requires this path to be in the SVN repo, prevent returning True
121 # for paths that are only locally-added.
122 ret = False
123 else:
124 # Don't consider files tracked as deleted in the WC as under source-control.
125 # Consider files which are locally added/copied as under source-control.
126 ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
127 ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
128 return ret
129
130 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
131 """
132 Given a source path, walk the SVN history backwards to inspect the ancestory of
133 that path, seeing if it traces back to base_path. Build an array of copyfrom_path
134 and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
135 which base_path is a substring match of (e.g. we crawled back to the initial branch-
136 copy from trunk), then return the collection of ancestor paths. Otherwise,
137 copyfrom_path has no ancestory compared to base_path.
138
139 This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
140 file/folder was renamed in a branch and then that branch was merged back to trunk.
141
142 'svn_repos_url' is the full URL to the root of the SVN repository,
143 e.g. 'file:///path/to/repo'
144 'base_path' is the path in the SVN repo to the target path we're trying to
145 trace ancestry back to, e.g. 'trunk'.
146 'source_path' is the path in the SVN repo to the source path to start checking
147 ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
148 (full_path = svn_repos_url+base_path+"/"+path_offset)
149 'source_rev' is the revision to start walking the history of source_path backwards from.
150 """
151 ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s base_path: %s",
152 svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
153 done = False
154 working_path = base_path+"/"+source_path
155 working_rev = source_rev
156 first_iter_done = False
157 ancestors_temp = []
158 while not done:
159 # Get the first "svn log" entry for this path (relative to @rev)
160 ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
161 log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path, 1, working_rev, True)
162 if not log_entry:
163 ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
164 done = True
165 break
166 # If we found a copy-from case which matches our base_path, we're done.
167 # ...but only if we've at least tried to search for the first copy-from path.
168 if first_iter_done and working_path.startswith(base_path):
169 ui.status(prefix + ">> find_svn_ancestors: Done: Found working_path.startswith(base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
170 done = True
171 break
172 first_iter_done = True
173 # Search for any actions on our target path (or parent paths).
174 changed_paths_temp = []
175 for d in log_entry['changed_paths']:
176 path = d['path']
177 if path in working_path:
178 changed_paths_temp.append({'path': path, 'data': d})
179 if not changed_paths_temp:
180 # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
181 ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
182 done = True
183 continue
184 # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
185 changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
186 # Find the action for our working_path in this revision. Use a loop to check in reverse order,
187 # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
188 for v in changed_paths:
189 d = v['data']
190 path = d['path']
191 # Check action-type for this file
192 action = d['action']
193 if action not in _valid_svn_actions:
194 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
195 % (log_entry['revision'], action))
196 ui.status(prefix + "> %s %s%s", action, path,
197 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
198 level=ui.DEBUG, color='YELLOW')
199 if action == 'D':
200 # If file/folder was deleted, it has no ancestor
201 ancestors_temp = []
202 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
203 done = True
204 break
205 if action in 'RA':
206 # If file/folder was added/replaced but not a copy, it has no ancestor
207 if not d['copyfrom_path']:
208 ancestors_temp = []
209 ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
210 "Added" if action == "A" else "Replaced",
211 level=ui.DEBUG, color='YELLOW')
212 done = True
213 break
214 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
215 # and keep checking for ancestors
216 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
217 action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
218 level=ui.DEBUG, color='YELLOW')
219 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
220 'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
221 working_path = working_path.replace(d['path'], d['copyfrom_path'])
222 working_rev = d['copyfrom_revision']
223 # Follow the copy and keep on searching
224 break
225 ancestors = []
226 if ancestors_temp:
227 ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
228 working_path = base_path+"/"+source_path
229 for idx in range(len(ancestors_temp)):
230 d = ancestors_temp[idx]
231 working_path = working_path.replace(d['path'], d['copyfrom_path'])
232 working_rev = d['copyfrom_rev']
233 ancestors.append({'path': working_path, 'revision': working_rev})
234 if ui.get_level() >= ui.DEBUG:
235 max_len = 0
236 for idx in range(len(ancestors)):
237 d = ancestors[idx]
238 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
239 ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
240 for idx in range(len(ancestors)-1):
241 d = ancestors[idx]
242 d_next = ancestors[idx+1]
243 ui.status(prefix + " [%s] %s <-- %s", idx,
244 str(d['path']+"@"+str(d['revision'])).ljust(max_len),
245 str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
246 level=ui.DEBUG, color='YELLOW')
247 else:
248 ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
249 svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
250 return ancestors
251
252 def get_rev_map(source_rev, prefix):
253 """
254 Find the equivalent rev # in the target repo for the given rev # from the source repo.
255 """
256 ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
257 # Find the highest entry less-than-or-equal-to source_rev
258 for rev in range(int(source_rev), 0, -1):
259 ui.status(prefix + ">> get_rev_map: rev=%s in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
260 if rev in rev_map:
261 return int(rev_map[rev])
262 # Else, we fell off the bottom of the rev_map. Ruh-roh...
263 return None
264
265 def set_rev_map(source_rev, target_rev):
266 ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
267 global rev_map
268 rev_map[int(source_rev)]=int(target_rev)
269
270 def build_rev_map(target_url, target_end_rev, source_info):
271 """
272 Check for any already-replayed history from source_url (source_info) and
273 build the mapping-table of source_rev -> target_rev.
274 """
275 global rev_map
276 rev_map = {}
277 ui.status("Rebuilding rev_map...", level=ui.VERBOSE)
278 proc_count = 0
279 it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
280 for log_entry in it_log_entries:
281 if log_entry['revprops']:
282 revprops = {}
283 for v in log_entry['revprops']:
284 if v['name'].startswith('svn2svn:'):
285 revprops[v['name']] = v['value']
286 if revprops and \
287 revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
288 revprops['svn2svn:source_url'] == source_info['url']:
289 source_rev = revprops['svn2svn:source_rev']
290 target_rev = log_entry['revision']
291 set_rev_map(source_rev, target_rev)
292
293 def get_svn_dirlist(svn_path, svn_rev = ""):
294 """
295 Get a list of all the child contents (recusive) of the given folder path.
296 """
297 args = ["list"]
298 path = svn_path
299 if svn_rev:
300 args += ["-r", svn_rev]
301 path += "@"+str(svn_rev)
302 args += [path]
303 paths = run_svn(args, no_fail=True)
304 paths = paths.strip("\n").split("\n") if len(paths)>1 else []
305 return paths
306
307 def path_in_list(paths, path):
308 for p in paths:
309 if path.startswith(p):
310 return True
311 return False
312
313 def add_path(paths, path):
314 if not path_in_list(paths, path):
315 paths.append(path)
316
317 def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom_rev="", \
318 export_paths={}, is_dir = False, prefix = ""):
319 """
320 Given the add'd source path, replay the "svn add/copy" commands to correctly
321 track renames across copy-from's.
322
323 For example, consider a sequence of events like this:
324 1. svn copy /trunk /branches/fix1
325 2. (Make some changes on /branches/fix1)
326 3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2 " Rename folder
327 4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt " Rename file inside renamed folder
328 5. svn co /trunk && svn merge /branches/fix1
329 After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
330 and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
331 to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
332 we'd lose the logical history that Proj2/file2.txt is really a descendant
333 of Proj1/file1.txt.
334
335 'path_offset' is the offset from source_base to the file to check ancestry for,
336 e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
337 'source_rev' is the revision ("svn log") that we're processing from the source repo.
338 'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
339 directory, when being called recursively by do_svn_add_dir().
340 'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
341 'is_dir' is whether path_offset is a directory (rather than a file).
342 """
343 ui.status(prefix + ">> do_svn_add: %s %s", source_base+"/"+path_offset+"@"+str(source_rev),
344 " (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
345 level=ui.DEBUG, color='GREEN')
346 # Check if the given path has ancestors which chain back to the current source_base
347 found_ancestor = False
348 ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+" ")
349 # ancestors[n] is the original (pre-branch-copy) trunk path.
350 # ancestors[n-1] is the first commit on the new branch.
351 copyfrom_path = ancestors[len(ancestors)-1]['path'] if ancestors else ""
352 copyfrom_rev = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
353 if ancestors:
354 # The copy-from path has ancestory back to source_url.
355 ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
356 level=ui.DEBUG, color='GREEN', bold=True)
357 found_ancestor = True
358 # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
359 # return None in the case where copyfrom_rev is *before* our source_start_rev.
360 tgt_rev = get_rev_map(copyfrom_rev, prefix+" ")
361 ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
362 else:
363 ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
364 found_ancestor = False
365 if found_ancestor and tgt_rev:
366 # Check if this path_offset in the target WC already has this ancestry, in which
367 # case there's no need to run the "svn copy" (again).
368 path_in_svn = in_svn(path_offset, prefix=prefix+" ")
369 log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+" ") else []
370 if (not log_entry or (log_entry['revision'] != tgt_rev)):
371 copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
372 ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
373 ui.status(prefix + " copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
374 ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
375 if path_in_svn and \
376 ((parent_copyfrom_path and copyfrom_path.startswith(parent_copyfrom_path)) and \
377 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
378 # When being called recursively, if this child entry has the same ancestor as the
379 # the parent, then no need to try to run another "svn copy".
380 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
381 parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
382 pass
383 else:
384 # Copy this path from the equivalent path+rev in the target repo, to create the
385 # equivalent history.
386 if parent_copyfrom_path:
387 # If we have a parent copy-from path, we mis-match that so display a status
388 # message describing the action we're mimic'ing. If path_in_svn, then this
389 # is logically a "replace" rather than an "add".
390 ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
391 if path_in_svn:
392 # If local file is already under version-control, then this is a replace.
393 ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
394 run_svn(["remove", "--force", path_offset])
395 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
396 # Export the final version of this file/folder from the source repo, to make
397 # sure we're up-to-date.
398 add_path(export_paths, path_offset)
399 else:
400 ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
401 else:
402 # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
403 # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
404 # Create (parent) directory if needed.
405 # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
406 # split-out to a shared tag?
407 p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
408 if not os.path.exists(p_path):
409 run_svn(["mkdir", p_path])
410 if not in_svn(path_offset, prefix=prefix+" "):
411 if is_dir:
412 # Export the final verison of all files in this folder.
413 add_path(export_paths, path_offset)
414 else:
415 # Export the final verison of this file. We *need* to do this before running
416 # the "svn add", even if we end-up re-exporting this file again via export_paths.
417 run_svn(["export", "--force", "-r", source_rev,
418 source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
419 # If not already under version-control, then "svn add" this file/folder.
420 run_svn(["add", "--parents", path_offset])
421 # TODO: Need to copy SVN properties from source repos
422 if is_dir:
423 # For any folders that we process, process any child contents, so that we correctly
424 # replay copies/replaces/etc.
425 do_svn_add_dir(path_offset, source_rev, copyfrom_path, copyfrom_rev, export_paths, prefix+" ")
426
427 def do_svn_add_dir(path_offset, source_rev, parent_copyfrom_path, parent_copyfrom_rev, \
428 export_paths, prefix=""):
429 # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
430 # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
431 # associated remote repo folder. (Is this a problem?)
432 paths_local = get_svn_dirlist(path_offset)
433 paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
434 ui.status(prefix + ">> do_svn_add_dir: paths_local: %s", str(paths_local), level=ui.DEBUG, color='GREEN')
435 ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
436 # Update files/folders which exist in remote but not local
437 for path in paths_remote:
438 path_is_dir = True if path[-1] == "/" else False
439 working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
440 do_svn_add(working_path, source_rev, parent_copyfrom_path, parent_copyfrom_rev,
441 export_paths, path_is_dir, prefix+" ")
442 # Remove files/folders which exist in local but not remote
443 for path in paths_local:
444 if not path in paths_remote:
445 ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
446 run_svn(["remove", "--force", path_offset+"/"+path])
447 # TODO: Does this handle deleted folders too? Wouldn't want to have a case
448 # where we only delete all files from folder but leave orphaned folder around.
449
450 def process_svn_log_entry(log_entry, options, commit_paths, prefix = ""):
451 """
452 Process SVN changes from the given log entry. Build an array (commit_paths)
453 of the paths in the working-copy that were changed, i.e. the paths which
454 we'll pass to "svn commit".
455 """
456 export_paths = []
457 source_rev = log_entry['revision']
458 ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
459 for d in log_entry['changed_paths']:
460 # Get the full path for this changed_path
461 # e.g. '/branches/bug123/projectA/file1.txt'
462 path = d['path']
463 if not path.startswith(source_base + "/"):
464 # Ignore changed files that are not part of this subdir
465 if path != source_base:
466 ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
467 continue
468 # Note: d['kind']="" for action="M" paths which only have property changes.
469 path_is_dir = True if d['kind'] == 'dir' else False
470 path_is_file = True if d['kind'] == 'file' else False
471 # Calculate the offset (based on source_base) for this changed_path
472 # e.g. 'projectA/file1.txt'
473 # (path = source_base + "/" + path_offset)
474 path_offset = path[len(source_base):].strip("/")
475 # Get the action for this path
476 action = d['action']
477 if action not in _valid_svn_actions:
478 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
479 % (source_rev, action))
480 ui.status(" %s %s%s", action, d['path'],
481 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
482 level=ui.VERBOSE)
483
484 # Try to be efficient and keep track of an explicit list of paths in the
485 # working copy that changed. If we commit from the root of the working copy,
486 # then SVN needs to crawl the entire working copy looking for pending changes.
487 add_path(commit_paths, path_offset)
488
489 # Special-handling for replace's
490 if action == 'R':
491 # If file was "replaced" (deleted then re-added, all in same revision),
492 # then we need to run the "svn rm" first, then change action='A'. This
493 # lets the normal code below handle re-"svn add"'ing the files. This
494 # should replicate the "replace".
495 run_svn(["remove", "--force", path_offset])
496 action = 'A'
497
498 # Handle all the various action-types
499 # (Handle "add" first, for "svn copy/move" support)
500 if action == 'A':
501 # Determine where to export from.
502 svn_copy = False
503 # Handle cases where this "add" was a copy from another URL in the source repos
504 if d['copyfrom_revision']:
505 copyfrom_path = d['copyfrom_path']
506 copyfrom_rev = d['copyfrom_revision']
507 do_svn_add(path_offset, source_rev, "", "", export_paths, path_is_dir, prefix+" ")
508 # Else just "svn export" the files from the source repo and "svn add" them.
509 else:
510 # Create (parent) directory if needed
511 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
512 if not os.path.exists(p_path):
513 run_svn(["mkdir", p_path])
514 # Export the entire added tree.
515 if path_is_dir:
516 # For directories, defer the (recurisve) "svn export". Might have a
517 # situation in a branch merge where the entry in the svn-log is a
518 # non-copy-from'd "add" but there are child contents (that we haven't
519 # gotten to yet in log_entry) that are copy-from's. When we try do
520 # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
521 # having pre-existing (svn-add'd) contents creates some trouble.
522 # Instead, just create the stub folders ("svn mkdir" above) and defer
523 # exporting the final file-state until the end.
524 add_path(export_paths, path_offset)
525 else:
526 # Export the final verison of this file. We *need* to do this before running
527 # the "svn add", even if we end-up re-exporting this file again via export_paths.
528 run_svn(["export", "--force", "-r", source_rev,
529 source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
530 if not in_svn(path_offset, prefix=prefix+" "):
531 # Need to use in_svn here to handle cases where client committed the parent
532 # folder and each indiv sub-folder.
533 run_svn(["add", "--parents", path_offset])
534 # TODO: Need to copy SVN properties from source repos
535
536 elif action == 'D':
537 run_svn(["remove", "--force", path_offset])
538
539 elif action == 'M':
540 # TODO: Is "svn merge -c" correct here? Should this just be an "svn export" plus
541 # proplist updating?
542 out = run_svn(["merge", "-c", source_rev, "--non-recursive",
543 "--non-interactive", "--accept=theirs-full",
544 source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
545 # TODO: If d['props'] == 'modified', then run code to clean-up/purge any newly-modified props?
546
547 else:
548 raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
549 % action)
550
551 # Export the final version of all add'd paths from source_url
552 if export_paths:
553 for path_offset in export_paths:
554 run_svn(["export", "--force", "-r", source_rev,
555 source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
556
557 def disp_svn_log_summary(log_entry):
558 ui.status("")
559 ui.status("r%s | %s | %s",
560 log_entry['revision'],
561 log_entry['author'],
562 str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
563 ui.status(log_entry['message'])
564 ui.status("------------------------------------------------------------------------")
565
566 def real_main(options, args):
567 global source_url, target_url, rev_map
568 source_url = args.pop(0).rstrip("/") # e.g. 'http://server/svn/source/trunk'
569 target_url = args.pop(0).rstrip("/") # e.g. 'file:///svn/target/trunk'
570 ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
571
572 # Make sure that both the source and target URL's are valid
573 source_info = svnclient.get_svn_info(source_url)
574 assert source_url.startswith(source_info['repos_url'])
575 target_info = svnclient.get_svn_info(target_url)
576 assert target_url.startswith(target_info['repos_url'])
577
578 # Init global vars
579 global source_repos_url,source_base,source_repos_uuid
580 source_repos_url = source_info['repos_url'] # e.g. 'http://server/svn/source'
581 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
582 source_repos_uuid = source_info['repos_uuid']
583
584 source_end_rev = source_info['revision'] # Last revision # in the source repo
585 target_end_rev = target_info['revision'] # Last revision # in the target repo
586 wc_target = os.path.abspath('_wc_target')
587 num_entries_proc = 0
588 commit_count = 0
589 source_rev = None
590 target_rev = None
591
592 # Check out a working copy of target_url if needed
593 wc_exists = os.path.exists(wc_target)
594 if wc_exists and not options.cont_from_break:
595 shutil.rmtree(wc_target)
596 wc_exists = False
597 if not wc_exists:
598 svnclient.svn_checkout(target_url, wc_target)
599 os.chdir(wc_target)
600
601 if not options.cont_from_break:
602 # TODO: Warn user if trying to start (non-continue) into a non-empty target path?
603 # Get log entry for the SVN revision we will check out
604 if options.svn_rev:
605 # If specify a rev, get log entry just before or at rev
606 source_start_log = svnclient.get_last_svn_log_entry(source_url, 1, options.svn_rev, False)
607 else:
608 # Otherwise, get log entry of branch creation
609 it_log_start = svnclient.iter_svn_log_entries(source_url, 1, source_end_rev, get_changed_paths=False)
610 for source_start_log in it_log_start:
611 break
612 if not source_start_log:
613 raise InternalError("Unable to find first revision for source_url: %s" % source_url)
614
615 # This is the revision we will start from for source_url
616 source_start_rev = source_rev = int(source_start_log['revision'])
617 ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
618
619 # For the initial commit to the target URL, export all the contents from
620 # the source URL at the start-revision.
621 paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)])
622 if len(paths)>1:
623 disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev))
624 ui.status("(Initial import)", level=ui.VERBOSE)
625 paths = paths.strip("\n").split("\n")
626 for path_raw in paths:
627 # For each top-level file/folder...
628 if not path_raw:
629 continue
630 # Directories have a trailing slash in the "svn list" output
631 path_is_dir = True if path_raw[-1] == "/" else False
632 path = path_raw.rstrip('/') if path_is_dir else path_raw
633 if path_is_dir and not os.path.exists(path):
634 os.makedirs(path)
635 ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path, level=ui.VERBOSE)
636 run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path+"@"+str(source_rev), path])
637 run_svn(["add", path])
638 num_entries_proc += 1
639 target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's
640 target_rev = commit_from_svn_log_entry(source_start_log, options, target_revprops=target_revprops)
641 if target_rev:
642 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
643 set_rev_map(source_rev, target_rev)
644 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
645 run_svn(["update"])
646 commit_count += 1
647 else:
648 # Re-build the rev_map based on any already-replayed history in target_url
649 build_rev_map(target_url, target_end_rev, source_info)
650 if not rev_map:
651 raise RuntimeError("Called with continue-mode, but no already-replayed history found in target repo: %s" % target_url)
652 source_start_rev = int(max(rev_map, key=rev_map.get))
653 assert source_start_rev
654 ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
655
656 svn_vers_t = svnclient.get_svn_client_version()
657 svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
658
659 # Load SVN log starting from source_start_rev + 1
660 it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True) if source_start_rev < source_end_rev else []
661 source_rev = None
662
663 try:
664 for log_entry in it_log_entries:
665 if options.entries_proc_limit:
666 if num_entries_proc >= options.entries_proc_limit:
667 break
668 # Replay this revision from source_url into target_url
669 disp_svn_log_summary(log_entry)
670 source_rev = log_entry['revision']
671 # Process all the changed-paths in this log entry
672 commit_paths = []
673 process_svn_log_entry(log_entry, options, commit_paths)
674 num_entries_proc += 1
675 # Commit any changes made to _wc_target
676 target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's
677 target_rev = commit_from_svn_log_entry(log_entry, options, commit_paths, target_revprops=target_revprops)
678 if target_rev:
679 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
680 source_rev = log_entry['revision']
681 set_rev_map(source_rev, target_rev)
682 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
683 run_svn(["update"])
684 commit_count += 1
685 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
686 if svn_vers >= 1.7 and (commit_count % 100 == 0):
687 run_svn(["cleanup"])
688 if not source_rev:
689 # If there were no new source_url revisions to process, init source_rev
690 # for the "finally" message below.
691 source_rev = source_end_rev
692
693 except KeyboardInterrupt:
694 print "\nStopped by user."
695 print "\nCleaning-up..."
696 run_svn(["cleanup"])
697 full_svn_revert()
698 except:
699 print "\nCommand failed with following error:\n"
700 traceback.print_exc()
701 print "\nCleaning-up..."
702 run_svn(["cleanup"])
703 print run_svn(["status"])
704 full_svn_revert()
705 finally:
706 print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
707
708 def main():
709 # Defined as entry point. Must be callable without arguments.
710 usage = "Usage: %prog [OPTIONS] source_url target_url"
711 description = """\
712 Replicate (replay) history from one SVN repository to another. Maintain
713 logical ancestry wherever possible, so that 'svn log' on the replayed
714 repo will correctly follow file/folder renames.
715
716 == Examples ==
717 Create a copy of only /trunk from source repo, starting at r5000
718 $ svnadmin create /svn/target
719 $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
720 $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
721 1. The target_url will be checked-out to ./_wc_target
722 2. The first commit to http://server/source/trunk at/after r5000 will be
723 exported & added into _wc_target
724 3. All revisions affecting http://server/source/trunk (starting at r5000)
725 will be replayed to _wc_target. Any add/copy/move/replaces that are
726 copy-from'd some path outside of /trunk (e.g. files renamed on a
727 /branch and branch was merged into /trunk) will correctly maintain
728 logical ancestry where possible.
729
730 Use continue-mode (-c) to pick-up where the last run left-off
731 $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
732 1. The target_url will be checked-out to ./_wc_target, if not already
733 checked-out
734 2. All new revisions affecting http://server/source/trunk starting from
735 the last replayed revision to file:///svn/target/trunk (based on the
736 svn2svn:* revprops) will be replayed to _wc_target, maintaining all
737 logical ancestry where possible."""
738 parser = optparse.OptionParser(usage, description=description,
739 formatter=HelpFormatter(), version="%prog "+str(full_version))
740 #parser.remove_option("--help")
741 #parser.add_option("-h", "--help", dest="show_help", action="store_true",
742 # help="show this help message and exit")
743 parser.add_option("-r", "--revision", type="int", dest="svn_rev", metavar="REV",
744 help="initial SVN revision to start source_url replay")
745 parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author", default=False,
746 help="maintain original 'Author' info from source repo")
747 parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
748 help="continue from previous break")
749 parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
750 help="maximum number of log entries to process")
751 parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
752 help="try processing next log entry but don't commit changes to "
753 "target working-copy (forces --limit=1)")
754 parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
755 help="enable additional output (use -vv or -vvv for more)")
756 parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
757 help="enable debugging output (same as -vvv)")
758 options, args = parser.parse_args()
759 if len(args) != 2:
760 parser.error("incorrect number of arguments")
761 if options.verbosity < 10:
762 # Expand multiple "-v" arguments to a real ui._level value
763 options.verbosity *= 10
764 if options.dry_run:
765 # When in dry-run mode, only try to process the next log_entry
766 options.entries_proc_limit = 1
767 ui.update_config(options)
768 return real_main(options, args)
769
770
771 if __name__ == "__main__":
772 sys.exit(main() or 0)