Hash :
6d936bef
Author :
Date :
2023-05-18T11:51:05
Restore sync_restricted_traces_to_cipd upload progress bar https://crrev.com/c/4538368 suppressed stdout for all cipd invocations to reduce noise. This restores stdout piping to the calling process for uploads only, as cipd shows a useful progress bar during uploads. Bug: angleproject:8137 Change-Id: Ida7e43bc55af4dda1be768c65784fe3bc0656955 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/4545407 Reviewed-by: Cody Northrop <cnorthrop@google.com> Commit-Queue: Cody Northrop <cnorthrop@google.com>
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
#!/usr/bin/python3
#
# Copyright 2021 The ANGLE Project Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# sync_restricted_traces_to_cipd.py:
# Ensures the restricted traces are uploaded to CIPD. Versions are encoded in
# restricted_traces.json. Requires access to the CIPD path to work.
import argparse
from concurrent import futures
import getpass
import fnmatch
import logging
import json
import os
import platform
import signal
import subprocess
import sys
CIPD_PREFIX = 'angle/traces'
EXPERIMENTAL_CIPD_PREFIX = 'experimental/google.com/%s/angle/traces'
LOG_LEVEL = 'info'
JSON_PATH = 'restricted_traces.json'
SCRIPT_DIR = os.path.dirname(sys.argv[0])
MAX_THREADS = 8
LONG_TIMEOUT = 100000
def cipd(args, suppress_stdout=True):
logging.debug('running cipd with args: %s', ' '.join(args))
exe = 'cipd.bat' if platform.system() == 'Windows' else 'cipd'
if suppress_stdout:
# Capture stdout, only log if --log=debug after the process terminates
process = subprocess.run([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if process.stdout:
logging.debug('cipd stdout:\n%s' % process.stdout.decode())
else:
# Stdout is piped to the caller's stdout, visible immediately
process = subprocess.run([exe] + args)
return process.returncode
def cipd_name_and_version(trace, trace_version):
if 'x' in trace_version:
trace_prefix = EXPERIMENTAL_CIPD_PREFIX % getpass.getuser()
trace_version = trace_version.strip('x')
else:
trace_prefix = CIPD_PREFIX
trace_name = '%s/%s' % (trace_prefix, trace)
return trace_name, trace_version
def check_trace_exists(args, trace, trace_version):
cipd_trace_name, cipd_trace_version = cipd_name_and_version(trace, trace_version)
# Determine if this version exists
return cipd(['describe', cipd_trace_name, '-version', 'version:%s' % cipd_trace_version]) == 0
def upload_trace(args, trace, trace_version):
trace_folder = os.path.join(SCRIPT_DIR, trace)
cipd_trace_name, cipd_trace_version = cipd_name_and_version(trace, trace_version)
cipd_args = ['create', '-name', cipd_trace_name]
cipd_args += ['-in', trace_folder]
cipd_args += ['-tag', 'version:%s' % cipd_trace_version]
cipd_args += ['-log-level', args.log.lower()]
cipd_args += ['-install-mode', 'copy']
if cipd(cipd_args, suppress_stdout=False) != 0:
logging.error('%s version %s: cipd create failed', trace, trace_version)
sys.exit(1)
logging.info('Uploaded trace to cipd: %s version:%s', cipd_trace_name, cipd_trace_version)
def check_trace_before_upload(trace):
for root, dirs, files in os.walk(os.path.join(SCRIPT_DIR, trace)):
if dirs:
logging.error('Sub-directories detected for trace %s: %s' % (trace, dirs))
sys.exit(1)
trace_json = trace + '.json'
with open(os.path.join(root, trace_json)) as f:
jtrace = json.load(f)
additional_files = set([trace_json, trace + '.angledata.gz'])
extra_files = set(files) - set(jtrace['TraceFiles']) - additional_files
if extra_files:
logging.error('Unexpected files, not listed in %s.json [TraceFiles]:\n%s', trace,
'\n'.join(extra_files))
sys.exit(1)
def main(args):
logging.basicConfig(level=args.log.upper())
with open(os.path.join(SCRIPT_DIR, JSON_PATH)) as f:
traces = json.loads(f.read())
logging.info('Checking cipd for existing versions (this takes time without --filter)')
f_exists = {}
trace_versions = {}
with futures.ThreadPoolExecutor(max_workers=args.threads) as executor:
for trace_info in traces['traces']:
trace, trace_version = trace_info.split(' ')
trace_versions[trace] = trace_version
if args.filter and not fnmatch.fnmatch(trace, args.filter):
logging.debug('Skipping %s because it does not match the test filter.' % trace)
continue
assert trace not in f_exists
f_exists[trace] = executor.submit(check_trace_exists, args, trace, trace_version)
to_upload = [trace for trace, f in f_exists.items() if not f.result()]
if not to_upload:
logging.info('All traces are in sync with cipd')
return 0
logging.info('The following traces are out of sync with cipd:')
for trace in to_upload:
print(' ', trace, trace_versions[trace])
check_trace_before_upload(trace)
if args.upload or input('Upload [y/N]?') == 'y':
for trace in to_upload:
upload_trace(args, trace, trace_versions[trace])
else:
logging.error('Aborted')
return 1
return 0
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument(
'-p', '--prefix', help='CIPD Prefix. Default: %s' % CIPD_PREFIX, default=CIPD_PREFIX)
parser.add_argument(
'-l', '--log', help='Logging level. Default: %s' % LOG_LEVEL, default=LOG_LEVEL)
parser.add_argument(
'-f', '--filter', help='Only sync specified tests. Supports fnmatch expressions.')
parser.add_argument(
'-t',
'--threads',
help='Maxiumum parallel threads. Default: %s' % MAX_THREADS,
default=MAX_THREADS)
parser.add_argument('--upload', action='store_true', help='Upload without asking.')
args = parser.parse_args()
logging.basicConfig(level=args.log.upper())
sys.exit(main(args))