From 24cf7d6bec4d377c6342badeb8dd91444e787efa Mon Sep 17 00:00:00 2001 From: Julian Maurice Date: Wed, 15 Nov 2023 11:17:49 +0100 Subject: [PATCH] Bug 35342: Use a Makefile to manage PO files There were some problems recently involving gulp and PO files, which are still unresolved at the time of writing. Debugging this is hard and time-consuming (failures are hard to reproduce) Why make/a Makefile ? * Creating/updating PO files is essentially passing a list of input files to a command that will produce a new/updated output file, and that's exactly what make does * It builds only what's needed by default (if source files haven't changed, no need to rebuild the .pot, and so .po files don't need to be updated either) * It handles parallelism well To use the Makefile, you need to be at the root directory of Koha, and then you can run: make -f misc/translator/Makefile help To have a list of available targets make -f misc/translator/Makefile all To update all PO files make -f misc/translator/Makefile pot To rebuild all POT files make -f misc/translator/Makefile en-GB To update all en-GB PO files make -f misc/translator/Makefile misc/translator/Koha-installer.pot To update that particular file By default, it won't build a target (output file) if its prerequisites (input files) are older than the target. It can be forced with -B/--always-make By default, all targets are built sequentially. To enable parallelism, use -j/--jobs Multiple targets can be specified on the command line, so it's possible to do: make -f misc/translator/Makefile -j -B en-GB fr-FR gulpfile.js is updated to uses this Makefile This patch also includes a fix in Koha/Database/Columns.pm (a translated empty string caused gettext to emit a warning and to place the header in the middle of the .po/.pot file) --- Koha/Database/Columns.pm | 2 +- gulpfile.js | 324 ++++----------------------------------- misc/translator/Makefile | 90 +++++++++++ 3 files changed, 122 insertions(+), 294 deletions(-) create mode 100644 misc/translator/Makefile diff --git a/Koha/Database/Columns.pm b/Koha/Database/Columns.pm index 4637d0cab3a..7f6b17b5dbd 100644 --- a/Koha/Database/Columns.pm +++ b/Koha/Database/Columns.pm @@ -306,7 +306,7 @@ sub columns { "datetime" => __("Statistics date and time"), "itemnumber" => __("Item number"), "itemtype" => __("Itemtype"), - "other" => __(""), + "other" => "", "type" => __("Type"), "value" => __("Value"), }, diff --git a/gulpfile.js b/gulpfile.js index 3e471a6f5e7..f5e3ef8f3f4 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -88,70 +88,14 @@ function staff_css(){ return css(STAFF_CSS_BASE); } -const poTasks = { - 'marc-MARC21': { - extract: po_extract_marc_marc21, - create: po_create_marc_marc21, - update: po_update_marc_marc21, - }, - 'marc-UNIMARC': { - extract: po_extract_marc_unimarc, - create: po_create_marc_unimarc, - update: po_update_marc_unimarc, - }, - 'staff-prog': { - extract: po_extract_staff, - create: po_create_staff, - update: po_update_staff, - }, - 'opac-bootstrap': { - extract: po_extract_opac, - create: po_create_opac, - update: po_update_opac, - }, - 'pref': { - extract: po_extract_pref, - create: po_create_pref, - update: po_update_pref, - }, - 'messages': { - extract: po_extract_messages, - create: po_create_messages, - update: po_update_messages, - }, - 'messages-js': { - extract: po_extract_messages_js, - create: po_create_messages_js, - update: po_update_messages_js, - }, - 'installer': { - extract: po_extract_installer, - create: po_create_installer, - update: po_update_installer, - }, - 'installer-MARC21': { - extract: po_extract_installer_marc21, - create: po_create_installer_marc21, - update: po_update_installer_marc21, - }, - 'installer-UNIMARC': { - extract: po_extract_installer_unimarc, - create: po_create_installer_unimarc, - update: po_update_installer_unimarc, - }, -}; - function getPoTasks () { - let tasks = []; - - let all_tasks = Object.keys(poTasks); - - if (args.task) { - tasks = [args.task].flat(Infinity); - } else { + const all_tasks = ['marc-MARC21', 'marc-UNIMARC', 'staff-prog', 'opac-bootstrap', 'pref', 'messages', 'messages-js', 'installer', 'installer-MARC21', 'installer-UNIMARC']; + if (!args.task) { return all_tasks; } + const tasks = [args.task].flat(Infinity); + let invalid_tasks = tasks.filter( function( el ) { return all_tasks.indexOf( el ) < 0; }); @@ -163,236 +107,6 @@ function getPoTasks () { return tasks; } -const poTypes = getPoTasks(); - -function po_extract_marc (type) { - return src(`koha-tmpl/*-tmpl/*/en/**/*${type}*`, { read: false, nocase: true }) - .pipe(xgettext('misc/translator/xgettext.pl --charset=UTF-8 -F', `Koha-marc-${type}.pot`)) - .pipe(dest('misc/translator')) -} - -function po_extract_marc_marc21 () { return po_extract_marc('MARC21') } -function po_extract_marc_unimarc () { return po_extract_marc('UNIMARC') } - -function po_extract_staff () { - const globs = [ - 'koha-tmpl/intranet-tmpl/prog/en/**/*.tt', - 'koha-tmpl/intranet-tmpl/prog/en/**/*.inc', - 'koha-tmpl/intranet-tmpl/prog/en/xslt/*.xsl', - '!koha-tmpl/intranet-tmpl/prog/en/**/*MARC21*', - '!koha-tmpl/intranet-tmpl/prog/en/**/*UNIMARC*', - '!koha-tmpl/intranet-tmpl/prog/en/**/*marc21*', - '!koha-tmpl/intranet-tmpl/prog/en/**/*unimarc*', - ]; - - return src(globs, { read: false, nocase: true }) - .pipe(xgettext('misc/translator/xgettext.pl --charset=UTF-8 -F', 'Koha-staff-prog.pot')) - .pipe(dest('misc/translator')) -} - -function po_extract_opac () { - const globs = [ - 'koha-tmpl/opac-tmpl/bootstrap/en/**/*.tt', - 'koha-tmpl/opac-tmpl/bootstrap/en/**/*.inc', - 'koha-tmpl/opac-tmpl/bootstrap/en/xslt/*.xsl', - '!koha-tmpl/opac-tmpl/bootstrap/en/**/*MARC21*', - '!koha-tmpl/opac-tmpl/bootstrap/en/**/*UNIMARC*', - '!koha-tmpl/opac-tmpl/bootstrap/en/**/*marc21*', - '!koha-tmpl/opac-tmpl/bootstrap/en/**/*unimarc*', - ]; - - return src(globs, { read: false, nocase: true }) - .pipe(xgettext('misc/translator/xgettext.pl --charset=UTF-8 -F', 'Koha-opac-bootstrap.pot')) - .pipe(dest('misc/translator')) -} - -const xgettext_options = '--from-code=UTF-8 --package-name Koha ' - + '--package-version= -k -k__ -k__x -k__n:1,2 -k__nx:1,2 -k__xn:1,2 ' - + '-k__p:1c,2 -k__px:1c,2 -k__np:1c,2,3 -k__npx:1c,2,3 -kN__ ' - + '-kN__n:1,2 -kN__p:1c,2 -kN__np:1c,2,3 ' - + '-k -k$__ -k$__x -k$__n:1,2 -k$__nx:1,2 -k$__xn:1,2 ' - + '--force-po'; - -function po_extract_messages_js () { - const globs = [ - 'koha-tmpl/intranet-tmpl/prog/js/vue/**/*.vue', - 'koha-tmpl/intranet-tmpl/prog/js/**/*.js', - 'koha-tmpl/opac-tmpl/bootstrap/js/**/*.js', - ]; - - return src(globs, { read: false, nocase: true }) - .pipe(xgettext(`xgettext -L JavaScript ${xgettext_options}`, 'Koha-messages-js.pot')) - .pipe(dest('misc/translator')) -} - -function po_extract_messages () { - const perlStream = src(['**/*.pl', '**/*.pm'], { read: false, nocase: true }) - .pipe(xgettext(`xgettext -L Perl ${xgettext_options}`, 'Koha-perl.pot')) - - const ttStream = src([ - 'koha-tmpl/intranet-tmpl/prog/en/**/*.tt', - 'koha-tmpl/intranet-tmpl/prog/en/**/*.inc', - 'koha-tmpl/opac-tmpl/bootstrap/en/**/*.tt', - 'koha-tmpl/opac-tmpl/bootstrap/en/**/*.inc', - ], { read: false, nocase: true }) - .pipe(xgettext('misc/translator/xgettext-tt2 --from-code=UTF-8', 'Koha-tt.pot')) - - const headers = { - 'Project-Id-Version': 'Koha', - 'Content-Type': 'text/plain; charset=UTF-8', - }; - - return merge(perlStream, ttStream) - .pipe(concatPo('Koha-messages.pot', { headers })) - .pipe(dest('misc/translator')) -} - -function po_extract_pref () { - return src('koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/*.pref', { read: false }) - .pipe(xgettext('misc/translator/xgettext-pref', 'Koha-pref.pot')) - .pipe(dest('misc/translator')) -} - -function po_extract_installer () { - const globs = [ - 'installer/data/mysql/en/mandatory/*.yml', - 'installer/data/mysql/en/optional/*.yml', - ]; - - return src(globs, { read: false, nocase: true }) - .pipe(xgettext('misc/translator/xgettext-installer', 'Koha-installer.pot')) - .pipe(dest('misc/translator')) -} - -function po_extract_installer_marc (type) { - const globs = `installer/data/mysql/en/marcflavour/${type}/**/*.yml`; - - return src(globs, { read: false, nocase: true }) - .pipe(xgettext('misc/translator/xgettext-installer', `Koha-installer-${type}.pot`)) - .pipe(dest('misc/translator')) -} - -function po_extract_installer_marc21 () { return po_extract_installer_marc('MARC21') } - -function po_extract_installer_unimarc () { return po_extract_installer_marc('UNIMARC') } - -function po_create_type (type) { - const access = util.promisify(fs.access); - const exec = util.promisify(child_process.exec); - - const pot = `misc/translator/Koha-${type}.pot`; - - // Generate .pot only if it doesn't exist or --force-extract is given - const extract = () => stream.finished(poTasks[type].extract()); - const p = - args['generate-pot'] === 'always' ? extract() : - args['generate-pot'] === 'auto' ? access(pot).catch(extract) : - args['generate-pot'] === 'never' ? Promise.resolve(0) : - Promise.reject(new Error('Invalid value for option --generate-pot: ' + args['generate-pot'])) - - return p.then(function () { - const languages = getLanguages(); - const promises = []; - for (const language of languages) { - const locale = language.split('-').filter(s => s.length !== 4).join('_'); - const po = `misc/translator/po/${language}-${type}.po`; - - const promise = access(po) - .catch(() => exec(`msginit -o ${po} -i ${pot} -l ${locale} --no-translator`)) - promises.push(promise); - } - - return Promise.all(promises); - }); -} - -function po_create_marc_marc21 () { return po_create_type('marc-MARC21') } -function po_create_marc_unimarc () { return po_create_type('marc-UNIMARC') } -function po_create_staff () { return po_create_type('staff-prog') } -function po_create_opac () { return po_create_type('opac-bootstrap') } -function po_create_pref () { return po_create_type('pref') } -function po_create_messages () { return po_create_type('messages') } -function po_create_messages_js () { return po_create_type('messages-js') } -function po_create_installer () { return po_create_type('installer') } -function po_create_installer_marc21 () { return po_create_type('installer-MARC21') } -function po_create_installer_unimarc () { return po_create_type('installer-UNIMARC') } - -function po_update_type (type) { - const access = util.promisify(fs.access); - const exec = util.promisify(child_process.exec); - - const pot = `misc/translator/Koha-${type}.pot`; - - // Generate .pot only if it doesn't exist or --force-extract is given - const extract = () => stream.finished(poTasks[type].extract()); - const p = - args['generate-pot'] === 'always' ? extract() : - args['generate-pot'] === 'auto' ? access(pot).catch(extract) : - args['generate-pot'] === 'never' ? Promise.resolve(0) : - Promise.reject(new Error('Invalid value for option --generate-pot: ' + args['generate-pot'])) - - return p.then(function () { - const languages = getLanguages(); - const promises = []; - for (const language of languages) { - const po = `misc/translator/po/${language}-${type}.po`; - promises.push(exec(`msgmerge --backup=off --no-wrap --quiet -F --update ${po} ${pot}`)); - } - - return Promise.all(promises); - }); -} - -function po_update_marc_marc21 () { return po_update_type('marc-MARC21') } -function po_update_marc_unimarc () { return po_update_type('marc-UNIMARC') } -function po_update_staff () { return po_update_type('staff-prog') } -function po_update_opac () { return po_update_type('opac-bootstrap') } -function po_update_pref () { return po_update_type('pref') } -function po_update_messages () { return po_update_type('messages') } -function po_update_messages_js () { return po_update_type('messages-js') } -function po_update_installer () { return po_update_type('installer') } -function po_update_installer_marc21 () { return po_update_type('installer-MARC21') } -function po_update_installer_unimarc () { return po_update_type('installer-UNIMARC') } - -/** - * Gulp plugin that executes xgettext-like command `cmd` on all files given as - * input, and then outputs the result as a POT file named `filename`. - * `cmd` should accept -o and -f options - */ -function xgettext (cmd, filename) { - const filenames = []; - - function transform (file, encoding, callback) { - filenames.push(path.relative(file.cwd, file.path)); - callback(); - } - - function flush (callback) { - fs.mkdtemp(path.join(os.tmpdir(), 'koha-'), (err, folder) => { - const outputFilename = path.join(folder, filename); - const filesFilename = path.join(folder, 'files'); - fs.writeFile(filesFilename, filenames.join(os.EOL), err => { - if (err) return callback(err); - - const command = `${cmd} -o ${outputFilename} -f ${filesFilename}`; - child_process.exec(command, err => { - if (err) return callback(err); - - fs.readFile(outputFilename, (err, data) => { - if (err) return callback(err); - - const file = new Vinyl(); - file.path = path.join(file.base, filename); - file.contents = data; - callback(null, file); - }); - }); - }); - }) - } - - return through2.obj(transform, flush); -} /** * Return languages selected for PO-related tasks @@ -428,6 +142,30 @@ exports.watch = function () { watch(STAFF_CSS_BASE + "/src/**/*.scss", series('staff_css')); }; -exports['po:create'] = parallel(...poTypes.map(type => poTasks[type].create)); -exports['po:update'] = parallel(...poTypes.map(type => poTasks[type].update)); -exports['po:extract'] = parallel(...poTypes.map(type => poTasks[type].extract)); +exports['po:create'] = function () { + const targets = getLanguages(); + const cmdargs = ['--makefile=misc/translator/Makefile', '--jobs']; + if (args['generate-pot'] === 'always') { + cmdargs.push('--always-make'); + } + cmdargs.push(`EXTRA_LANGS=${targets.join(' ')}`) + + return child_process.spawn('make', cmdargs.concat(targets), { stdio: 'inherit' }); +} + +exports['po:update'] = function () { + const targets = getLanguages(); + const cmdargs = ['--makefile=misc/translator/Makefile', '--jobs']; + if (args['generate-pot'] === 'always') { + cmdargs.push('--always-make'); + } + + return child_process.spawn('make', cmdargs.concat(targets), { stdio: 'inherit' }); +} + +exports['po:extract'] = function () { + const targets = getPoTasks().map(type => `misc/translator/Koha-${type}.pot`); + const cmdargs = ['--makefile=misc/translator/Makefile', '--jobs', '--always-make']; + + return child_process.spawn('make', cmdargs.concat(targets), { stdio: 'inherit' }); +}; diff --git a/misc/translator/Makefile b/misc/translator/Makefile new file mode 100644 index 00000000000..1df2baba7a7 --- /dev/null +++ b/misc/translator/Makefile @@ -0,0 +1,90 @@ +MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +MAKEFILE_DIR := $(dir $(MAKEFILE_PATH)) +SRC_DIR := $(abspath $(MAKEFILE_DIR)/../..) +ifneq ($(SRC_DIR),$(CURDIR)) +$(error This makefile should be used from Koha root directory. Use make -C $(SRC_DIR) -f $(MAKEFILE_PATH) $(MAKECMDGOALS)) +endif + +define help +Available targets: + all + Update all PO files + pot + Rebuild all POT files + clean + Delete all POT files + , eg. en-GB + Update all PO files for that language + , eg. misc/translator/po/en-GB-installer.po or misc/translator/Koha-installer.pot + Update that particular file +endef + +TYPES := installer installer-MARC21 installer-UNIMARC marc-MARC21 marc-UNIMARC messages messages-js opac-bootstrap pref staff-prog +LANGS := $(patsubst %-installer.po,%,$(notdir $(wildcard misc/translator/po/*-installer.po))) +LANGS += $(EXTRA_LANGS) +POT_DIR := misc/translator +PO_DIR := misc/translator/po +POTS := $(foreach TYPE,$(TYPES),$(POT_DIR)/Koha-$(TYPE).pot) +XGETTEXT_COMMON_OPTS := --sort-output --from-code=UTF-8 --package-name Koha --package-version= --force-po -k -k__ -k__x -k__n:1,2 -k__nx:1,2 -k__xn:1,2 -k__p:1c,2 -k__px:1c,2 -k__np:1c,2,3 -k__npx:1c,2,3 -kN__ -kN__n:1,2 -kN__p:1c,2 -kN__np:1c,2,3 +XGETTEXT_PERL_OPTS := $(XGETTEXT_COMMON_OPTS) -L Perl +XGETTEXT_JS_OPTS := $(XGETTEXT_COMMON_OPTS) -L JavaScript -k$$__ -k$$__x -k$$__n:1,2 -k$$__nx:1,2 -k$$__xn:1,2 + +installer_DEPS := $(shell find installer/data/mysql/en/mandatory installer/data/mysql/en/optional -iname '*.yml' | sort) +installer_CMD := misc/translator/xgettext-installer +installer-MARC21_DEPS := $(shell find installer/data/mysql/en/marcflavour/marc21 -iname '*.yml' | sort) +installer-MARC21_CMD := misc/translator/xgettext-installer +installer-UNIMARC_DEPS := $(shell find installer/data/mysql/en/marcflavour/unimarc -iname '*.yml' | sort) +installer-UNIMARC_CMD := misc/translator/xgettext-installer +marc-MARC21_DEPS := $(shell find koha-tmpl/opac-tmpl/bootstrap/en -iname '*MARC21*' | sort) $(shell find koha-tmpl/intranet-tmpl/prog/en -iname '*MARC21*'| sort -d) +marc-MARC21_CMD := misc/translator/xgettext.pl --charset=UTF-8 +marc-UNIMARC_DEPS := $(shell find koha-tmpl/opac-tmpl/bootstrap/en -iname '*UNIMARC*' | sort) $(shell find koha-tmpl/intranet-tmpl/prog/en -iname '*UNIMARC*'| sort) +marc-UNIMARC_CMD := misc/translator/xgettext.pl --charset=UTF-8 +messages-perl_DEPS := $(shell find -iname '*.pl' -or -iname '*.pm' | sort) +messages-perl_CMD := xgettext $(XGETTEXT_PERL_OPTS) +messages-tt_DEPS := $(shell find koha-tmpl/opac-tmpl/bootstrap/en koha-tmpl/intranet-tmpl/prog/en -iname *.tt -or -iname *.inc | sort) +messages-tt_CMD := misc/translator/xgettext-tt2 +messages_DEPS := misc/translator/Koha-messages-perl.pot misc/translator/Koha-messages-tt.pot +messages_CMD := msgcat --use-first +messages-js_DEPS := $(shell find koha-tmpl/intranet-tmpl/prog/js koha-tmpl/opac-tmpl/bootstrap/js -iname '*.js' -or -iname '*.vue' | sort) +messages-js_CMD := xgettext $(XGETTEXT_JS_OPTS) +opac-bootstrap_DEPS := $(shell find koha-tmpl/opac-tmpl/bootstrap/en \( -iname '*.tt' -or -iname '*.inc' -or -iname '*.xsl' \) -not -iname '*MARC21*' -not -iname '*UNIMARC*' | sort) +opac-bootstrap_CMD := misc/translator/xgettext.pl --charset=UTF-8 +pref_DEPS := $(shell find koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences -iname '*.pref' | sort) +pref_CMD := misc/translator/xgettext-pref +staff-prog_DEPS := $(shell find koha-tmpl/intranet-tmpl/prog/en \( -iname '*.tt' -or -iname '*.inc' -or -iname '*.xsl' \) -not -iname '*MARC21*' -not -iname '*UNIMARC*' | sort) +staff-prog_CMD := misc/translator/xgettext.pl --charset=UTF-8 + +.PHONY: all help pot clean $(LANGS) +all: $(LANGS) +help:; @$(info $(help)) : + +pot: $(POTS) + +clean: + rm -f misc/translator/*.pot + +$(LANGS): %: $(foreach TYPE,$(TYPES),$(PO_DIR)/%-$(TYPE).po) + +define pot-rule-template +misc/translator/Koha-$(1).pot: $$($(1)_DEPS) + @echo "Extract: $$@" + $$(file >$$@.in) $$(foreach O,$$^,$$(file >>$$@.in,$$O)) + @$$($(1)_CMD) -o $$@.tmp -f $$@.in + @msgcat --sort-by-file --no-wrap -o $$@ $$@.tmp + @rm $$@.tmp $$@.in +endef +$(foreach TYPE,$(TYPES),$(eval $(call pot-rule-template,$(TYPE)))) +$(foreach TYPE,messages-perl messages-tt,$(eval $(call pot-rule-template,$(TYPE)))) + +define po-rule-template +$$(foreach LANG,$$(LANGS),$$(PO_DIR)/$$(LANG)-$(1).po): $$(POT_DIR)/Koha-$(1).pot + @if test -f $$@; then \ + echo "Update: $$@"; \ + msgmerge --backup=off --quiet --sort-by-file --update $$@ $$< && touch "$$@"; \ + else \ + echo "Create: $$@"; \ + locale=$$$$(basename $$@ .po | perl -nE 'say join("_", grep { (length) < 4 } (split /-/))'); \ + msginit --no-translator -o $$@ -i $$< -l $$$$locale ; \ + fi +endef +$(foreach TYPE,$(TYPES),$(eval $(call po-rule-template,$(TYPE)))) -- 2.30.2