Re: [PATCH] Packager.py should take advantage of the parallelism from the package manager.
Lucas Meneghel Rodrigues <lmr <at> redhat.com>
2011-03-02 00:50:47 GMT
On Tue, 2011-02-22 at 13:01 -0800, Jean-Marc Eurin wrote:
> Packager.py should take advantage of the parallelism from the package manager.
> Create the repo directory if it doesn't exists.
Applied, thanks!
http://autotest.kernel.org/changeset/5273
> Signed-off-by: Jean-Marc Eurin <jmeurin <at> google.com>
>
> --- autotest/client/common_lib/base_packages.py 2011-02-09 14:25:38.000000000 -0800
> +++ autotest/client/common_lib/base_packages.py 2011-02-22 12:51:10.000000000 -0800
> <at> <at> -28,27 +28,36 <at> <at>
> "Incorrect SSH path in global_config: %s" % repo)
>
>
> -def repo_run_command(repo, cmd, ignore_status=False):
> +def repo_run_command(repo, cmd, ignore_status=False, cd=True):
> """Run a command relative to the repos path"""
> repo = repo.strip()
> run_cmd = None
> + cd_str = ''
> if repo.startswith('ssh://'):
> username = None
> hostline, remote_path = parse_ssh_path(repo)
> + if cd:
> + cd_str = 'cd %s && ' % remote_path
> if ' <at> ' in hostline:
> username, host = hostline.split(' <at> ')
> - run_cmd = 'ssh %s <at> %s "cd %s && %s"' % (username, host,
> - remote_path, cmd)
> + run_cmd = 'ssh %s <at> %s "%s%s"' % (username, host, cd_str, cmd)
> else:
> - run_cmd = 'ssh %s "cd %s && %s"' % (host, remote_path, cmd)
> + run_cmd = 'ssh %s "%s%s"' % (host, cd_str, cmd)
>
> else:
> - run_cmd = "cd %s && %s" % (repo, cmd)
> + if cd:
> + cd_str = 'cd %s && ' % repo
> + run_cmd = "%s%s" % (cd_str, cmd)
>
> if run_cmd:
> return utils.run(run_cmd, ignore_status=ignore_status)
>
>
> +def create_directory(repo):
> + _, remote_path = parse_ssh_path(repo)
> + repo_run_command(repo, 'mkdir -p %s' % remote_path, cd=False)
> +
> +
> def check_diskspace(repo, min_free=None):
> # Note: 1 GB = 10**9 bytes (SI unit).
> if not min_free:
> <at> <at> -272,6 +281,7 <at> <at>
> if not repo.startswith('/') and not repo.startswith('ssh:'):
> return
> try:
> + create_directory(repo)
> check_diskspace(repo)
> check_write(repo)
> except (error.RepoWriteError, error.RepoUnknownError,
> --- autotest/utils/packager.py 2009-10-13 17:18:06.000000000 -0700
> +++ autotest/utils/packager.py 2011-02-22 12:51:10.000000000 -0800
> <at> <at> -71,7 +71,7 <at> <at>
>
>
> # Method to upload or remove package depending on the flag passed to it.
> -def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, repo_url,
> +def process_packages(pkgmgr, pkg_type, pkg_names, src_dir,
> remove=False):
> exclude_string = ' .'
> names = [p.strip() for p in pkg_names.split(',')]
> <at> <at> -101,12 +101,12 <at> <at>
> raise error.RepoDiskFullError(msg)
> tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
> temp_dir, exclude_string)
> - pkgmgr.upload_pkg(tarball_path, repo_url, update_checksum=True)
> + pkgmgr.upload_pkg(tarball_path, update_checksum=True)
> finally:
> # remove the temporary directory
> shutil.rmtree(temp_dir)
> else:
> - pkgmgr.remove_pkg(pkg_name, repo_url, remove_checksum=True)
> + pkgmgr.remove_pkg(pkg_name, remove_checksum=True)
> print "Done."
>
>
> <at> <at> -137,7 +137,7 <at> <at>
> return tarballs
>
>
> -def process_all_packages(pkgmgr, client_dir, upload_paths, remove=False):
> +def process_all_packages(pkgmgr, client_dir, remove=False):
> """Process a full upload of packages as a directory upload."""
> test_dir = os.path.join(client_dir, "tests")
> site_test_dir = os.path.join(client_dir, "site_tests")
> <at> <at> -185,22 +185,16 <at> <at>
> os.chdir(temp_dir)
> client_utils.system('md5sum * > packages.checksum')
> os.chdir(cwd)
> - for path in upload_paths:
> - print "Uploading to: " + path
> - pkgmgr.upload_pkg(temp_dir, path)
> + pkgmgr.upload_pkg(temp_dir)
> client_utils.run('rm -rf ' + temp_dir)
> else:
> - for repo_url in upload_paths:
> - process_packages(pkgmgr, 'test', tests, client_dir, repo_url,
> - remove=remove)
> - process_packages(pkgmgr, 'test', site_tests, client_dir, repo_url,
> - remove=remove)
> - process_packages(pkgmgr, 'client', 'autotest', client_dir, repo_url,
> - remove=remove)
> - process_packages(pkgmgr, 'dep', deps, dep_dir, repo_url,
> - remove=remove)
> - process_packages(pkgmgr, 'profiler', profilers, prof_dir, repo_url,
> - remove=remove)
> + process_packages(pkgmgr, 'test', tests, client_dir,remove=remove)
> + process_packages(pkgmgr, 'test', site_tests, client_dir, remove=remove)
> + process_packages(pkgmgr, 'client', 'autotest', client_dir,
> + remove=remove)
> + process_packages(pkgmgr, 'dep', deps, dep_dir, remove=remove)
> + process_packages(pkgmgr, 'profiler', profilers, prof_dir,
> + remove=remove)
>
>
> # Get the list of sub directories present in a directory
> <at> <at> -241,10 +235,6 <at> <at>
> if len(upload_paths) == 0:
> return
>
> - pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls,
> - upload_paths=upload_paths,
> - run_function_dargs={'timeout':600})
> -
> client_dir = os.path.join(autotest_dir, "client")
>
> # Bail out if the client directory does not exist
> <at> <at> -267,35 +257,39 <at> <at>
> # we should not be getting here
> assert(False)
>
> + if options.repo:
> + upload_path_list = [options.repo]
> + else:
> + upload_path_list = upload_paths
> +
> + pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls,
> + upload_paths=upload_path_list,
> + run_function_dargs={'timeout':600})
> +
> if options.all:
> - if options.repo:
> - upload_path_list = [options.repo]
> - else:
> - upload_path_list = upload_paths
> - process_all_packages(pkgmgr, client_dir, upload_path_list,
> - remove=remove_flag)
> + process_all_packages(pkgmgr, client_dir, remove=remove_flag)
>
> if options.client:
> process_packages(pkgmgr, 'client', 'autotest', client_dir,
> - options.repo, remove=remove_flag)
> + remove=remove_flag)
>
> if options.dep:
> process_packages(pkgmgr, 'dep', options.dep, dep_dir,
> - options.repo, remove=remove_flag)
> + remove=remove_flag)
>
> if options.test:
> process_packages(pkgmgr, 'test', options.test, client_dir,
> - options.repo, remove=remove_flag)
> + remove=remove_flag)
>
> if options.prof:
> process_packages(pkgmgr, 'profiler', options.prof, prof_dir,
> - options.repo, remove=remove_flag)
> + remove=remove_flag)
>
> if options.file:
> if remove_flag:
> - pkgmgr.remove_pkg(options.file, options.repo, remove_checksum=True)
> + pkgmgr.remove_pkg(options.file, remove_checksum=True)
> else:
> - pkgmgr.upload_pkg(options.file, options.repo, update_checksum=True)
> + pkgmgr.upload_pkg(options.file, update_checksum=True)
>
>
> if __name__ == "__main__":
> _______________________________________________
> Autotest mailing list
> Autotest <at> test.kernel.org
> http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
RSS Feed