From 6c9cda1635859571de5c964bbacdece824045305 Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Fri, 27 Nov 2020 05:44:21 -0800 Subject: [vcpkg] Implement VersionedPortfileProvider and BaselineProvider (#14123) * WIP: Get versions from database files * Fix formatting * Provider inherits ResourceBase * Correct versions JSON file location * Fix formatting * Fix formatting * Fix include in versions.h * Fetch port versions using git tree object * Undo changes to x-history * Remove unnecesary moves Co-authored-by: nicole mazzuca * Extract Git manipulation code * [WIP] Review comments * [WIP] Review comments pt. 2 * [WIP] Review comments / fix formatting * Generate baseline.json * Extract deserializers from registries source file * BaselineProvider initial implementation * Modify gitignore * Update .gitignore again * Use JSON deserializer for versions db * Lazy load baseline file * Fetch baseline.json from baseline commit * More git abstractions * Clean up code * Path helpers * Formatting * Move data into impl object * Use implementation object for VersionedPortfileProvider * Reuse cloned instance for checkouts * Code cleanup and formatting * Fix returning dangling reference * Prepare to remove files in port_versions/ * Remove files in port_versions/ * Update .gitignore * Some PR review comments * Use StringView * More StringView conversions * More refactoring * Make some implementation members private * Functions for parsing baseline and version files * Hide deserializers implementation * Check for `versions` feature flag in registries. Co-authored-by: Robert Schumacher Co-authored-by: nicole mazzuca --- scripts/generatePortVersionsDb.py | 71 +++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 14 deletions(-) (limited to 'scripts') diff --git a/scripts/generatePortVersionsDb.py b/scripts/generatePortVersionsDb.py index cefd61e1c..e3c338c64 100644 --- a/scripts/generatePortVersionsDb.py +++ b/scripts/generatePortVersionsDb.py @@ -4,6 +4,7 @@ import sys import subprocess import json import time +import shutil from subprocess import CalledProcessError from json.decoder import JSONDecodeError @@ -14,9 +15,9 @@ SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) def get_current_git_ref(): - output = subprocess.run(['git.exe', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'], - capture_output=True, - encoding='utf-8') + output = subprocess.run(['git', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'], + capture_output=True, + encoding='utf-8') if output.returncode == 0: return output.stdout.strip() print(f"Failed to get git ref:", output.stderr.strip(), file=sys.stderr) @@ -25,47 +26,89 @@ def get_current_git_ref(): def generate_port_versions_db(ports_path, db_path, revision): start_time = time.time() - port_names = [item for item in os.listdir(ports_path) if os.path.isdir(os.path.join(ports_path, item))] + + # Assume each directory in ${VCPKG_ROOT}/ports is a different port + port_names = [item for item in os.listdir( + ports_path) if os.path.isdir(os.path.join(ports_path, item))] + port_names.sort() total_count = len(port_names) + + # Dictionary to collect the latest version of each port as baseline + baseline_objects = {} + baseline_objects['default'] = {} + for counter, port_name in enumerate(port_names): containing_dir = os.path.join(db_path, f'{port_name[0]}-') os.makedirs(containing_dir, exist_ok=True) + output_filepath = os.path.join(containing_dir, f'{port_name}.json') - if not os.path.exists(output_filepath): + if not os.path.exists(output_filepath): output = subprocess.run( - [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'), 'x-history', port_name, '--x-json'], + [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'), + 'x-history', port_name, '--x-json'], capture_output=True, encoding='utf-8') + if output.returncode == 0: try: versions_object = json.loads(output.stdout) + + # Put latest version in baseline dictionary + latest_version = versions_object["versions"][0] + baseline_objects['default'][port_name] = { + "version-string": latest_version["version-string"], + "port-version": latest_version["port-version"] + } with open(output_filepath, 'w') as output_file: json.dump(versions_object, output_file) except JSONDecodeError: - print(f'Maformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr) + print( + f'Malformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr) else: - print(f'x-history {port_name} failed: ', output.stdout.strip(), file=sys.stderr) + print(f'x-history {port_name} failed: ', + output.stdout.strip(), file=sys.stderr) + # This should be replaced by a progress bar if counter > 0 and counter % 100 == 0: elapsed_time = time.time() - start_time - print(f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds') + print( + f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds') + + # Generate baseline.json + baseline_file_path = os.path.join(db_path, 'baseline.json') + with open(baseline_file_path, 'w') as baseline_output_file: + json.dump(baseline_objects, baseline_output_file) + + # Generate timestamp rev_file = os.path.join(db_path, revision) Path(rev_file).touch() + elapsed_time = time.time() - start_time - print(f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds') - + print( + f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds') + def main(ports_path, db_path): revision = get_current_git_ref() if not revision: print('Couldn\'t fetch current Git revision', file=sys.stderr) sys.exit(1) + rev_file = os.path.join(db_path, revision) if os.path.exists(rev_file): print(f'Database files already exist for commit {revision}') sys.exit(0) - generate_port_versions_db(ports_path=ports_path, db_path=db_path, revision=revision) + + if (os.path.exists(db_path)): + try: + shutil.rmtree(db_path) + except OSError as e: + print(f'Could not delete folder: {db_path}.\nError: {e.strerror}') + + generate_port_versions_db(ports_path=ports_path, + db_path=db_path, + revision=revision) if __name__ == "__main__": - main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'), - db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions')) + main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'), + db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions')) -- cgit v1.2.3