Skip to content

Commit 613f2ee

Browse files
committed
Serialize bconded BuildRequires to a json file
Load it from the file when running jobs.py (This can be suppressed by `--no-cache` option) Merge cache on update (only update newer entries). Remove items from cache if they are not defined in config.toml (prevents data rotting). Assisted-By: claude 4.5 sonnet
1 parent fbe7608 commit 613f2ee

3 files changed

Lines changed: 3898 additions & 2 deletions

File tree

bconds.py

Lines changed: 123 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import argparse
22
import datetime
33
import functools
4+
import json
45
import os
56
import pathlib
67
import sys
@@ -291,6 +292,125 @@ def build_reverse_id_lookup():
291292
pass
292293

293294

295+
def serialize_bcond_config(bcond_config):
296+
"""
297+
Convert bcond_config to JSON-serializable dict.
298+
Handles Path → str and tuple → list conversions.
299+
"""
300+
serialized = {}
301+
302+
for key in ('withs', 'withouts', 'replacements'):
303+
if key in bcond_config:
304+
serialized[key] = bcond_config[key]
305+
306+
serialized['id'] = bcond_config['id']
307+
308+
# Store only SRPM filename (not full path, as paths differ between users)
309+
if 'srpm' in bcond_config:
310+
serialized['srpm_filename'] = bcond_config['srpm'].name
311+
312+
# Convert tuple to list for JSON
313+
if 'buildrequires' in bcond_config:
314+
serialized['buildrequires'] = list(bcond_config['buildrequires'])
315+
316+
return serialized
317+
318+
319+
def deserialize_bcond_config(config_dict):
320+
"""
321+
Convert JSON dict back to bcond_config format.
322+
Reconstructs Path and converts list → tuple.
323+
"""
324+
bcond_config = {}
325+
326+
for key in ('withs', 'withouts', 'replacements', 'id'):
327+
if key in config_dict:
328+
bcond_config[key] = config_dict[key]
329+
330+
if 'srpm_filename' in config_dict:
331+
repopath = pathlib.Path(CONFIG['cache_dir']['fedpkg']) / config_dict['id']
332+
srpm_path = repopath / config_dict['srpm_filename']
333+
# Only set if file exists (user may not have downloaded it)
334+
if srpm_path.exists():
335+
bcond_config['srpm'] = srpm_path
336+
337+
# Convert list to tuple (rpm_requires returns tuple for caching)
338+
if 'buildrequires' in config_dict:
339+
bcond_config['buildrequires'] = tuple(config_dict['buildrequires'])
340+
341+
return bcond_config
342+
343+
344+
def save_bconds_cache(filename='bconds_cache.json'):
345+
"""
346+
Save bcond configurations with BuildRequires to JSON file.
347+
Only saves configs that have 'buildrequires' populated.
348+
"""
349+
350+
all_bconds = {}
351+
for component_name, bcond_configs in CONFIG['bconds'].items():
352+
configs_with_br = [
353+
serialize_bcond_config(cfg)
354+
for cfg in bcond_configs
355+
if 'buildrequires' in cfg
356+
]
357+
if configs_with_br:
358+
all_bconds[component_name] = configs_with_br
359+
360+
cache_data = {
361+
'bconds': all_bconds
362+
}
363+
364+
try:
365+
with open(filename, 'w') as f:
366+
json.dump(cache_data, f, indent=2)
367+
except Exception as e:
368+
log(f'Warning: Failed to save bconds cache: {e}')
369+
370+
371+
def load_bconds_cache(filename='bconds_cache.json'):
372+
"""
373+
Load bcond configurations from JSON file into CONFIG['bconds'].
374+
Returns number of configs loaded.
375+
"""
376+
try:
377+
with open(filename) as f:
378+
cache_data = json.load(f)
379+
except json.JSONDecodeError as e:
380+
raise ValueError(f'Invalid JSON in cache file: {e}')
381+
382+
loaded_count = 0
383+
for component_name, config_dicts in cache_data.get('bconds', {}).items():
384+
if component_name not in CONFIG['bconds']:
385+
log(f'Warning: Component {component_name} in cache but not in config.toml, skipping')
386+
continue
387+
388+
# Match cached configs to existing configs by ID
389+
for config_dict in config_dicts:
390+
deserialized = deserialize_bcond_config(config_dict)
391+
392+
for existing_config in CONFIG['bconds'][component_name]:
393+
if existing_config.get('id') == deserialized['id']:
394+
existing_config.update(deserialized)
395+
loaded_count += 1
396+
break
397+
398+
return loaded_count
399+
400+
401+
def read_bconds_cache_if_exists():
402+
try:
403+
build_reverse_id_lookup()
404+
loaded_count = load_bconds_cache()
405+
log(f'Loaded {loaded_count} bcond configs from bconds_cache.json')
406+
except FileNotFoundError:
407+
log('Cache file bconds_cache.json not found, continuing without...')
408+
pass
409+
except ValueError as e:
410+
log(f'Error loading cache: {e}')
411+
sys.exit(1)
412+
413+
294414
def parse_args():
295415
parser = argparse.ArgumentParser()
296416
parser.add_argument(
@@ -332,8 +452,10 @@ def parse_args():
332452
something_was_downloaded |= download_srpm_if_possible(bcond_config)
333453
if extract_buildrequires_if_possible(bcond_config):
334454
extracted_count += 1
455+
save_bconds_cache()
335456
koji_status.cache_clear()
336457

337-
log(f'Extracted BuildRequires from {extracted_count} SRPMs.')
458+
log(f'Extracted BuildRequires from {extracted_count} SRPMs and saved to bconds_cache.json.')
459+
338460
if not_extracted_count := sum(len(bcond_configs) for bcond_configs in CONFIG['bconds'].values()) - extracted_count:
339461
sys.exit(f'{not_extracted_count} SRPMs remain to be built/downloaded/extracted, run this again in a while.')

0 commit comments

Comments
 (0)