amprolla is an apt repository merger originally intended for use with the Devuan infrastructure. This version is the third iteration of the software.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

123 lines
3.9 KiB

#!/usr/bin/env python3
# See LICENSE file for copyright and license details.
"""
Amprolla module for merging Translation files
"""
from bz2 import open as bz2_open
from multiprocessing import Pool
from os import makedirs
from os.path import dirname, join, isfile
import sys
import lib.globalvars as globalvars
from lib.config import (arches, categories, cpunm, i18n, mergedir, mergesubdir,
repos, spooldir)
from lib.lock import run_with_args_locking
from lib.log import die, info
from lib.parse import parse_packages
from amprolla_merge import prepare_merge_dict
def parse_translations_file(filename):
"""
Decompresses and parses a Translations file and returns a dict
"""
tfile = bz2_open(filename).read()
tfile = tfile.decode('utf-8')
return parse_packages(tfile)
def merge_translations(filelist):
"""
Merges a list of Translation files and returns a dict of the merged files
"""
merged_trnsl = {}
if not filelist:
return merged_trnsl
# This is determined in lib.config.repo_order
devuan_translation = filelist[0]
for f in reversed(filelist):
# Iteration order matters because we use .update on the dictionary.
# We use reverse lib.config.repo_order so Devuan is merged last and
# therefore has priority.
if f is None:
continue
elif isfile(f):
trnsl = parse_translations_file(f)
merged_trnsl.update(trnsl)
elif merged_trnsl and f == devuan_translation:
# In the absence of Devuan translations for forked packages, just
# remove upstream ones in case they are incorrect.
trnsl = parse_translations_file(join(dirname(f),
'Translation-en.bz2'))
print("%s not found: removing upstream translations of forked packages" % f)
for k in trnsl.keys():
merged_trnsl.pop(k, None)
return merged_trnsl
def write_translations(translations, filename):
"""
Writes a merged Translations dict to the given filename in bzip2 format
"""
makedirs(dirname(filename), exist_ok=True)
bz2f = bz2_open(filename, 'w')
for pkg, fields in sorted(translations.items()):
order = ['Package','Description-md5']
for key in sorted((k for k in fields.keys()
if k == 'Package' or k.startswith('Description-')),
key=lambda k: order.index(k) if k in order else len(order)+1):
line = "%s: %s\n" % (key, fields[key])
bz2f.write(line.encode('utf-8'))
bz2f.write(b'\n')
bz2f.close()
def main_merge(translation_file):
"""
Main merge logic. First parses the files into dictionaries, and
writes them to the mergedir afterwards
"""
to_merge = prepare_merge_dict()
for suite in to_merge:
globalvars.suite = suite
transl_list = []
for rep in to_merge[suite]:
if rep:
transl_list.append(join(rep, translation_file))
else:
transl_list.append(None)
print("Merging translations: %s" % transl_list)
translations_dict = merge_translations(transl_list)
if not translations_dict:
continue
outfile = transl_list[0].replace(join(spooldir,
repos['devuan']['dists']),
join(mergedir, mergesubdir))
print("Writing translations: %s" % outfile)
write_translations(translations_dict, outfile)
def main():
"""
Main function to allow multiprocessing.
"""
transl = []
for component in categories:
for i in i18n:
transl.append('{}/i18n/Translation-{}.bz2'.format(component, i))
mrgpool = Pool(cpunm)
mrgpool.map(main_merge, transl)
mrgpool.close()
if __name__ == '__main__':
run_with_args_locking(main, "translations merge")