From 0c4a8f217665e7f959add4eae0b8476843d18204 Mon Sep 17 00:00:00 2001 From: Tony Duckles Date: Wed, 11 Apr 2012 19:33:46 -0500 Subject: [PATCH] Use "--ignore-externals" for "svn update" and "svn export" * svn2svn/svnclient.py (update/remove/export): Introduce wrapper functions for update/remove/export. For update and remove, pass "--ignore-externals". * svn2svn/svnclient.py: Rename private functions to have a leading underscore. * svn2svn/svnclient.py: Rename public functions to match svn sub-command name: get_svn_info -> info, get_svn_status -> status, get_prop_val -> propget, etc. * svn2svn/run/svn2svn.py (verify_commit): Remove extraneous safe_path() call. svnclient.run_svn_log() already handles this. --- svn2svn/run/svn2svn.py | 59 +++++++++++------------- svn2svn/svnclient.py | 102 +++++++++++++++++++++++++++-------------- 2 files changed, 94 insertions(+), 67 deletions(-) diff --git a/svn2svn/run/svn2svn.py b/svn2svn/run/svn2svn.py index 3b928d6..08492b9 100644 --- a/svn2svn/run/svn2svn.py +++ b/svn2svn/run/svn2svn.py @@ -195,8 +195,8 @@ def verify_commit(source_rev, target_rev, log_entry=None): if count % 500 == 0: ui.status("...processed %s (%s of %s)..." % (count, count, count_total), level=ui.VERBOSE) ui.status("verify_commit: path_offset:%s", path_offset, level=ui.DEBUG, color='YELLOW') - source_log_entries = svnclient.run_svn_log(svnclient.safe_path(source_url.rstrip("/")+"/"+path_offset), source_rev, 1, source_rev-source_rev_first+1) - target_log_entries = svnclient.run_svn_log(svnclient.safe_path(target_url.rstrip("/")+"/"+path_offset), target_rev, 1, target_rev) + source_log_entries = svnclient.run_svn_log(source_url.rstrip("/")+"/"+path_offset, source_rev, 1, source_rev-source_rev_first+1) + target_log_entries = svnclient.run_svn_log(target_url.rstrip("/")+"/"+path_offset, target_rev, 1, target_rev) # Build a list of commits in source_log_entries which matches our # target path_offset. working_path = source_base+"/"+path_offset @@ -255,8 +255,8 @@ def verify_commit(source_rev, target_rev, log_entry=None): is_diff = True if sum1 <> sum2 else False if not is_diff: # Check for property changes - props1 = svnclient.get_all_props(source_repos_url+working_path, source_rev_tmp) - props2 = svnclient.get_all_props(source_repos_url+working_path_next, source_rev_tmp-1) + props1 = svnclient.propget_all(source_repos_url+working_path, source_rev_tmp) + props2 = svnclient.propget_all(source_repos_url+working_path_next, source_rev_tmp-1) # Ignore changes to "svn:mergeinfo", since we don't copy that if 'svn:mergeinfo' in props1: del props1['svn:mergeinfo'] if 'svn:mergeinfo' in props2: del props2['svn:mergeinfo'] @@ -380,8 +380,8 @@ def sync_svn_props(source_url, source_rev, path_offset): Carry-forward any unversioned properties from the source repo to the target WC. """ - source_props = svnclient.get_all_props(join_path(source_url, path_offset), source_rev) - target_props = svnclient.get_all_props(path_offset) + source_props = svnclient.propget_all(join_path(source_url, path_offset), source_rev) + target_props = svnclient.propget_all(path_offset) if 'svn:mergeinfo' in source_props: # Never carry-forward "svn:mergeinfo" del source_props['svn:mergeinfo'] @@ -403,7 +403,7 @@ def in_svn(p, require_in_repo=False, prefix=""): With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy. Use "svn status" to check the status of the file/folder. """ - entries = svnclient.get_svn_status(p, no_recursive=True) + entries = svnclient.status(p, non_recursive=True) if not entries: return False d = entries[0] @@ -702,16 +702,15 @@ def do_svn_add(source_url, path_offset, source_rev, source_ancestors, \ if path_in_svn: # If local file is already under version-control, then this is a replace. ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN') - run_svn(["update", svnclient.safe_path(path_offset)]) - run_svn(["remove", "--force", svnclient.safe_path(path_offset)]) + svnclient.update(path_offset) + svnclient.remove(path_offset, force=True) run_svn(["copy", "-r", tgt_rev, svnclient.safe_path(join_path(target_url, copyfrom_offset), tgt_rev), svnclient.safe_path(path_offset)]) if is_dir: # Export the final verison of all files in this folder. add_path(export_paths, path_offset) else: # Export the final verison of this file. - run_svn(["export", "--force", "-r", source_rev, - svnclient.safe_path(source_repos_url+join_path(source_base, path_offset), source_rev), svnclient.safe_path(path_offset)]) + svnclient.export(source_repos_url+join_path(source_base, path_offset), source_rev, path_offset, force=True) if options.keep_prop: sync_svn_props(source_url, source_rev, path_offset) else: @@ -732,8 +731,7 @@ def do_svn_add(source_url, path_offset, source_rev, source_ancestors, \ else: # Export the final verison of this file. We *need* to do this before running # the "svn add", even if we end-up re-exporting this file again via export_paths. - run_svn(["export", "--force", "-r", source_rev, - svnclient.safe_path(source_repos_url+join_path(source_base, path_offset), source_rev), svnclient.safe_path(path_offset)]) + svnclient.export(source_repos_url+join_path(source_base, path_offset), source_rev, path_offset, force=True) # If not already under version-control, then "svn add" this file/folder. run_svn(["add", "--parents", svnclient.safe_path(path_offset)]) if options.keep_prop: @@ -770,8 +768,8 @@ def do_svn_add_dir(source_url, path_offset, source_rev, source_ancestors, \ path_is_dir = True if path[-1] == "/" else False working_path = join_path(path_offset, (path.rstrip('/') if path_is_dir else path)).lstrip('/') ui.status(" %s %s", 'D', join_path(source_base, working_path), level=ui.VERBOSE) - run_svn(["update", svnclient.safe_path(working_path)]) - run_svn(["remove", "--force", svnclient.safe_path(working_path)]) + svnclient.update(working_path) + svnclient.remove(working_path, force=True) # TODO: Does this handle deleted folders too? Wouldn't want to have a case # where we only delete all files from folder but leave orphaned folder around. @@ -835,8 +833,8 @@ def process_svn_log_entry(log_entry, ancestors, commit_paths, prefix = ""): if path_is_dir: # Need to "svn update" before "svn remove" in case child contents are at # a higher rev than the (parent) path_offset. - run_svn(["update", svnclient.safe_path(path_offset)]) - run_svn(["remove", "--force", svnclient.safe_path(path_offset)]) + svnclient.update(path_offset) + svnclient.remove(path_offset, force=True) action = 'A' # Handle all the various action-types @@ -881,8 +879,7 @@ def process_svn_log_entry(log_entry, ancestors, commit_paths, prefix = ""): else: # Export the final verison of this file. We *need* to do this before running # the "svn add", even if we end-up re-exporting this file again via export_paths. - run_svn(["export", "--force", "-r", source_rev, - svnclient.safe_path(join_path(source_url, path_offset), source_rev), svnclient.safe_path(path_offset)]) + svnclient.export(join_path(source_url, path_offset), source_rev, path_offset, force=True) if not in_svn(path_offset, prefix=prefix+" "): # Need to use in_svn here to handle cases where client committed the parent # folder and each indiv sub-folder. @@ -895,20 +892,19 @@ def process_svn_log_entry(log_entry, ancestors, commit_paths, prefix = ""): # For dirs, need to "svn update" before "svn remove" because the final # "svn commit" will fail if the parent (path_offset) is at a lower rev # than any of the child contents. This needs to be a recursive update. - run_svn(["update", svnclient.safe_path(path_offset)]) - run_svn(["remove", "--force", svnclient.safe_path(path_offset)]) + svnclient.update(path_offset) + svnclient.remove(path_offset, force=True) elif action == 'M': if path_is_file: - run_svn(["export", "--force", "-N" , "-r", source_rev, - svnclient.safe_path(join_path(source_url, path_offset), source_rev), svnclient.safe_path(path_offset)]) + svnclient.export(join_path(source_url, path_offset), source_rev, path_offset, force=True, non_recursive=True) if path_is_dir: # For dirs, need to "svn update" before export/prop-sync because the # final "svn commit" will fail if the parent is at a lower rev than # child contents. Just need to update the rev-state of the dir (d['path']), # don't need to recursively update all child contents. # (??? is this the right reason?) - run_svn(["update", "-N", svnclient.safe_path(path_offset)]) + svnclient.update(path_offset, non_recursive=True) if options.keep_prop: sync_svn_props(source_url, source_rev, path_offset) @@ -919,8 +915,7 @@ def process_svn_log_entry(log_entry, ancestors, commit_paths, prefix = ""): # Export the final version of all add'd paths from source_url if export_paths: for path_offset in export_paths: - run_svn(["export", "--force", "-r", source_rev, - svnclient.safe_path(join_path(source_url, path_offset), source_rev), svnclient.safe_path(path_offset)]) + svnclient.export(join_path(source_url, path_offset), source_rev, path_offset, force=True) def keep_revnum(source_rev, target_rev_last, wc_target_tmp): """ @@ -973,9 +968,9 @@ def real_main(args): ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN') # Make sure that both the source and target URL's are valid - source_info = svnclient.get_svn_info(source_url) + source_info = svnclient.info(source_url) assert is_child_path(source_url, source_info['repos_url']) - target_info = svnclient.get_svn_info(target_url) + target_info = svnclient.info(target_url) assert is_child_path(target_url, target_info['repos_url']) # Init global vars @@ -989,12 +984,12 @@ def real_main(args): # Init start and end revision try: - source_start_rev = svnclient.get_svn_rev(source_repos_url, options.rev_start if options.rev_start else 1) + source_start_rev = svnclient.get_rev(source_repos_url, options.rev_start if options.rev_start else 1) except ExternalCommandFailed: print "Error: Invalid start source revision value: %s" % (options.rev_start) sys.exit(1) try: - source_end_rev = svnclient.get_svn_rev(source_repos_url, options.rev_end if options.rev_end else "HEAD") + source_end_rev = svnclient.get_rev(source_repos_url, options.rev_end if options.rev_end else "HEAD") except ExternalCommandFailed: print "Error: Invalid end source revision value: %s" % (options.rev_end) sys.exit(1) @@ -1073,7 +1068,7 @@ def real_main(args): raise InternalError("Cannot replay history on top of pre-existing structure: %s" % join_path(source_start_url, path_offset)) if path_is_dir and not os.path.exists(path_offset): os.makedirs(path_offset) - run_svn(["export", "--force", "-r" , source_start_rev, svnclient.safe_path(join_path(source_start_url, path_offset), source_start_rev), svnclient.safe_path(path_offset)]) + svnclient.export(join_path(source_start_url, path_offset), source_start_rev, path_offset, force=True) run_svn(["add", svnclient.safe_path(path_offset)]) # Update any properties on the newly added content paths = run_svn(["list", "--recursive", "-r", source_start_rev, svnclient.safe_path(source_start_url, source_start_rev)]) @@ -1111,7 +1106,7 @@ def real_main(args): ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE) ui.status("", level=ui.VERBOSE) - svn_vers_t = svnclient.get_svn_client_version() + svn_vers_t = svnclient.version() svn_vers = float(".".join(map(str, svn_vers_t[0:2]))) # Load SVN log starting from source_start_rev + 1 diff --git a/svn2svn/svnclient.py b/svn2svn/svnclient.py index f77a75a..d09e8a8 100644 --- a/svn2svn/svnclient.py +++ b/svn2svn/svnclient.py @@ -26,7 +26,7 @@ _forbidden_xml_chars = "".join( ) -def strip_forbidden_xml_chars(xml_string): +def _strip_forbidden_xml_chars(xml_string): """ Given an XML string, strips forbidden characters as per the XML spec. (these are all control characters except 0x9, 0xA and 0xD). @@ -48,7 +48,7 @@ def safe_path(path, rev_number=None): path += "@" return path -def svn_date_to_timestamp(svn_date): +def _svn_date_to_timestamp(svn_date): """ Parse an SVN date as read from the XML output and return the corresponding timestamp. @@ -60,13 +60,13 @@ def svn_date_to_timestamp(svn_date): time_tuple = time.strptime(date, "%Y-%m-%dT%H:%M:%S") return calendar.timegm(time_tuple) -def parse_svn_info_xml(xml_string): +def _parse_svn_info_xml(xml_string): """ Parse the XML output from an "svn info" command and extract useful information as a dict. """ d = {} - xml_string = strip_forbidden_xml_chars(xml_string) + xml_string = _strip_forbidden_xml_chars(xml_string) tree = ET.fromstring(xml_string) entry = tree.find('.//entry') d['url'] = entry.find('url').text @@ -78,7 +78,7 @@ def parse_svn_info_xml(xml_string): author_element = tree.find('.//commit/author') if author_element is not None: d['last_changed_author'] = author_element.text - d['last_changed_date'] = svn_date_to_timestamp(tree.find('.//commit/date').text) + d['last_changed_date'] = _svn_date_to_timestamp(tree.find('.//commit/date').text) # URL-decode "url" and "repos_url" values, since all paths passed # to run_svn() should be filtered through safe_path() and we don't # want to *double* URL-encode paths which are constructed used these values. @@ -113,16 +113,16 @@ def get_kind(svn_repos_url, svn_path, svn_rev, action, paths): # If no parent copy-from's, then we should be able to check this path in # the preceeding revision. info_rev -= 1 - info = get_svn_info(svn_repos_url+info_path, info_rev) - return info['kind'] + svn_info = info(svn_repos_url+info_path, info_rev) + return svn_info['kind'] -def parse_svn_log_xml(xml_string): +def _parse_svn_log_xml(xml_string): """ Parse the XML output from an "svn log" command and extract useful information as a list of dicts (one per log changeset). """ l = [] - xml_string = strip_forbidden_xml_chars(xml_string) + xml_string = _strip_forbidden_xml_chars(xml_string) tree = ET.fromstring(xml_string) for entry in tree.findall('logentry'): d = {} @@ -138,7 +138,7 @@ def parse_svn_log_xml(xml_string): msg = entry.find('msg') d['author'] = author is not None and author.text or "No author" d['date_raw'] = date.text if date is not None else None - d['date'] = svn_date_to_timestamp(date.text) if date is not None else None + d['date'] = _svn_date_to_timestamp(date.text) if date is not None else None d['message'] = msg is not None and msg.text and msg.text.replace('\r\n', '\n').replace('\n\r', '\n').replace('\r', '\n') or "" paths = [] for path in entry.findall('.//paths/path'): @@ -162,7 +162,7 @@ def parse_svn_log_xml(xml_string): l.append(d) return l -def parse_svn_status_xml(xml_string, base_dir=None, ignore_externals=False): +def _parse_svn_status_xml(xml_string, base_dir=None, ignore_externals=False): """ Parse the XML output from an "svn status" command and extract useful info as a list of dicts (one per status entry). @@ -170,7 +170,7 @@ def parse_svn_status_xml(xml_string, base_dir=None, ignore_externals=False): if base_dir: base_dir = os.path.normcase(base_dir) l = [] - xml_string = strip_forbidden_xml_chars(xml_string) + xml_string = _strip_forbidden_xml_chars(xml_string) tree = ET.fromstring(xml_string) for entry in tree.findall('.//entry'): d = {} @@ -197,26 +197,26 @@ def parse_svn_status_xml(xml_string, base_dir=None, ignore_externals=False): l.append(d) return l -def get_svn_rev(svn_url_or_wc, rev_number): +def get_rev(svn_url_or_wc, rev_number): """ Evaluate a given SVN revision pattern, to map it to a discrete rev #. """ xml_string = run_svn(['info', '--xml', '-r', rev_number, safe_path(svn_url_or_wc, rev_number)], fail_if_stderr=True) - info = parse_svn_info_xml(xml_string) + info = _parse_svn_info_xml(xml_string) return info['revision'] -def get_svn_info(svn_url_or_wc, rev_number=None): +def info(svn_url_or_wc, rev_number=None): """ Get SVN information for the given URL or working copy, with an optionally specified revision number. - Returns a dict as created by parse_svn_info_xml(). + Returns a dict as created by _parse_svn_info_xml(). """ args = ['info', '--xml'] if rev_number is not None: args += ["-r", rev_number] args += [safe_path(svn_url_or_wc, rev_number)] xml_string = run_svn(args, fail_if_stderr=True) - return parse_svn_info_xml(xml_string) + return _parse_svn_info_xml(xml_string) def svn_checkout(svn_url, checkout_dir, rev_number=None): """ @@ -242,9 +242,9 @@ def run_svn_log(svn_url_or_wc, rev_start, rev_end, limit, stop_on_copy=False, ge args += ['-r', '%s:%s' % (rev_start, rev_end)] args += ['--limit', str(limit), safe_path(svn_url_or_wc, max(rev_start, rev_end))] xml_string = run_svn(args) - return parse_svn_log_xml(xml_string) + return _parse_svn_log_xml(xml_string) -def get_svn_status(svn_wc, quiet=False, no_recursive=False): +def status(svn_wc, quiet=False, non_recursive=False): """ Get SVN status information about the given working copy. """ @@ -255,17 +255,17 @@ def get_svn_status(svn_wc, quiet=False, no_recursive=False): args += ['-q'] else: args += ['-v'] - if no_recursive: + if non_recursive: args += ['-N'] xml_string = run_svn(args + [safe_path(svn_wc)]) - return parse_svn_status_xml(xml_string, svn_wc, ignore_externals=True) + return _parse_svn_status_xml(xml_string, svn_wc, ignore_externals=True) def get_svn_versioned_files(svn_wc): """ Get the list of versioned files in the SVN working copy. """ contents = [] - for e in get_svn_status(svn_wc): + for e in status(svn_wc): if e['path'] and e['type'] == 'normal': contents.append(e['path']) return contents @@ -325,11 +325,11 @@ def iter_svn_log_entries(svn_url, first_rev, last_rev, stop_on_copy=False, get_c Use run/svn2svn.py:find_svn_ancestors() to pass in the 'ancestors' array so that we can correctly re-trace ancestry here. """ - info = get_svn_info(svn_url) - svn_repos_url = info['repos_url'] + svn_info = info(svn_url) + svn_repos_url = svn_info['repos_url'] #print "iter_svn_log_entries: %s %s:%s" % (svn_url, first_rev, last_rev) if last_rev == "HEAD": - last_rev = info['revision'] + last_rev = svn_info['revision'] if int(first_rev) == 1: start_log = get_first_svn_log_entry(svn_url, first_rev, last_rev, stop_on_copy=stop_on_copy, get_changed_paths=False) if start_log['revision'] > first_rev: @@ -397,7 +397,7 @@ def iter_svn_log_entries(svn_url, first_rev, last_rev, stop_on_copy=False, get_c _svn_client_version = None -def get_svn_client_version(): +def version(): """ Returns the SVN client version as a tuple. @@ -412,32 +412,32 @@ def get_svn_client_version(): return _svn_client_version -def parse_svn_propget_xml(xml_string): +def _parse_svn_propget_xml(xml_string): """ Parse the XML output from an "svn propget" command and extract useful information as a dict. """ d = {} - xml_string = strip_forbidden_xml_chars(xml_string) + xml_string = _strip_forbidden_xml_chars(xml_string) tree = ET.fromstring(xml_string) prop = tree.find('.//property') d['name'] = prop.get('name') d['value'] = prop is not None and prop.text and prop.text.replace('\r\n', '\n').replace('\n\r', '\n').replace('\r', '\n') or "" return d -def parse_svn_proplist_xml(xml_string): +def _parse_svn_proplist_xml(xml_string): """ Parse the XML output from an "svn proplist" command and extract list of property-names. """ l = [] - xml_string = strip_forbidden_xml_chars(xml_string) + xml_string = _strip_forbidden_xml_chars(xml_string) tree = ET.fromstring(xml_string) for prop in tree.findall('.//property'): l.append(prop.get('name')) return l -def get_prop_value(svn_url_or_wc, prop_name, rev_number=None): +def propget(svn_url_or_wc, prop_name, rev_number=None): """ Get the value of a versioned property for the given path. """ @@ -446,9 +446,9 @@ def get_prop_value(svn_url_or_wc, prop_name, rev_number=None): args += ['-r', rev_number] args += [prop_name, safe_path(svn_url_or_wc, rev_number)] xml_string = run_svn(args) - return parse_svn_propget_xml(xml_string) + return _parse_svn_propget_xml(xml_string) -def get_all_props(svn_url_or_wc, rev_number=None): +def propget_all(svn_url_or_wc, rev_number=None): """ Get the values of all versioned properties for the given path. """ @@ -458,8 +458,40 @@ def get_all_props(svn_url_or_wc, rev_number=None): args += ['-r', rev_number] args += [safe_path(svn_url_or_wc, rev_number)] xml_string = run_svn(args) - props = parse_svn_proplist_xml(xml_string) + props = _parse_svn_proplist_xml(xml_string) for prop_name in props: - d = get_prop_value(svn_url_or_wc, prop_name, rev_number) + d = propget(svn_url_or_wc, prop_name, rev_number) l[d['name']] = d['value'] return l + +def update(path, non_recursive=False): + """ + Update a path in a working-copy. + """ + args = ['update', '--ignore-externals'] + if non_recursive: + args += ['-N'] + args += [safe_path(path)] + run_svn(args) + +def remove(path, force=False): + """ + Remove a file/directory in a working-copy. + """ + args = ['remove'] + if force: + args += ['--force'] + args += [safe_path(path)] + run_svn(args) + +def export(svn_url, rev_number, path, non_recursive=False, force=False): + """ + Export a file from a repo to a local path. + """ + args = ['export', '--ignore-externals', '-r', rev_number] + if non_recursive: + args += ['-N'] + if force: + args += ['--force'] + args += [safe_path(svn_url, rev_number), safe_path(path)] + run_svn(args) -- 2.43.0