diff options
Diffstat (limited to 'scripts/generatePortVersionsDb.py')
| -rw-r--r-- | scripts/generatePortVersionsDb.py | 103 |
1 files changed, 38 insertions, 65 deletions
diff --git a/scripts/generatePortVersionsDb.py b/scripts/generatePortVersionsDb.py index e3c338c64..3b7de6942 100644 --- a/scripts/generatePortVersionsDb.py +++ b/scripts/generatePortVersionsDb.py @@ -1,17 +1,19 @@ import os
-import os.path
import sys
import subprocess
import json
import time
import shutil
-from subprocess import CalledProcessError
-from json.decoder import JSONDecodeError
+import multiprocessing
+
from pathlib import Path
+MAX_PROCESSES = multiprocessing.cpu_count()
SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
+PORTS_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, '../ports')
+VERSIONS_DB_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, '../port_versions')
def get_current_git_ref():
@@ -24,91 +26,62 @@ def get_current_git_ref(): return None
-def generate_port_versions_db(ports_path, db_path, revision):
+def generate_port_versions_file(port_name):
+ containing_dir = os.path.join(VERSIONS_DB_DIRECTORY, f'{port_name[0]}-')
+ os.makedirs(containing_dir, exist_ok=True)
+
+ output_file_path = os.path.join(containing_dir, f'{port_name}.json')
+ if not os.path.exists(output_file_path):
+ env = os.environ.copy()
+ env['GIT_OPTIONAL_LOCKS'] = '0'
+ output = subprocess.run(
+ [os.path.join(SCRIPT_DIRECTORY, '../vcpkg.exe'),
+ 'x-history', port_name, '--x-json', f'--output={output_file_path}'],
+ capture_output=True, encoding='utf-8', env=env)
+ if output.returncode != 0:
+ print(f'x-history {port_name} failed: ',
+ output.stdout.strip(), file=sys.stderr)
+
+
+def generate_port_versions_db(revision):
start_time = time.time()
# Assume each directory in ${VCPKG_ROOT}/ports is a different port
port_names = [item for item in os.listdir(
- ports_path) if os.path.isdir(os.path.join(ports_path, item))]
- port_names.sort()
+ PORTS_DIRECTORY) if os.path.isdir(os.path.join(PORTS_DIRECTORY, item))]
total_count = len(port_names)
- # Dictionary to collect the latest version of each port as baseline
- baseline_objects = {}
- baseline_objects['default'] = {}
-
- for counter, port_name in enumerate(port_names):
- containing_dir = os.path.join(db_path, f'{port_name[0]}-')
- os.makedirs(containing_dir, exist_ok=True)
-
- output_filepath = os.path.join(containing_dir, f'{port_name}.json')
- if not os.path.exists(output_filepath):
- output = subprocess.run(
- [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'),
- 'x-history', port_name, '--x-json'],
- capture_output=True, encoding='utf-8')
-
- if output.returncode == 0:
- try:
- versions_object = json.loads(output.stdout)
-
- # Put latest version in baseline dictionary
- latest_version = versions_object["versions"][0]
- baseline_objects['default'][port_name] = {
- "version-string": latest_version["version-string"],
- "port-version": latest_version["port-version"]
- }
- with open(output_filepath, 'w') as output_file:
- json.dump(versions_object, output_file)
- except JSONDecodeError:
- print(
- f'Malformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr)
- else:
- print(f'x-history {port_name} failed: ',
- output.stdout.strip(), file=sys.stderr)
-
- # This should be replaced by a progress bar
- if counter > 0 and counter % 100 == 0:
- elapsed_time = time.time() - start_time
- print(
- f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds')
-
- # Generate baseline.json
- baseline_file_path = os.path.join(db_path, 'baseline.json')
- with open(baseline_file_path, 'w') as baseline_output_file:
- json.dump(baseline_objects, baseline_output_file)
+ concurrency = MAX_PROCESSES / 2
+ print(f'Running {concurrency:.0f} parallel processes')
+ process_pool = multiprocessing.Pool(MAX_PROCESSES)
+ for i, _ in enumerate(process_pool.imap_unordered(generate_port_versions_file, port_names), 1):
+ sys.stderr.write(
+ f'\rProcessed: {i}/{total_count} ({(i / total_count):.2%})')
+ process_pool.close()
+ process_pool.join()
# Generate timestamp
- rev_file = os.path.join(db_path, revision)
+ rev_file = os.path.join(VERSIONS_DB_DIRECTORY, revision)
Path(rev_file).touch()
elapsed_time = time.time() - start_time
print(
- f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds')
+ f'\nElapsed time: {elapsed_time:.2f} seconds')
-def main(ports_path, db_path):
+def main():
revision = get_current_git_ref()
if not revision:
print('Couldn\'t fetch current Git revision', file=sys.stderr)
sys.exit(1)
- rev_file = os.path.join(db_path, revision)
+ rev_file = os.path.join(VERSIONS_DB_DIRECTORY, revision)
if os.path.exists(rev_file):
print(f'Database files already exist for commit {revision}')
sys.exit(0)
- if (os.path.exists(db_path)):
- try:
- shutil.rmtree(db_path)
- except OSError as e:
- print(f'Could not delete folder: {db_path}.\nError: {e.strerror}')
-
- generate_port_versions_db(ports_path=ports_path,
- db_path=db_path,
- revision=revision)
+ generate_port_versions_db(revision)
if __name__ == "__main__":
- main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'),
- db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions'))
+ main()
|