From 47c00f28baa8f8c72bed37728d61ffa042bca232 Mon Sep 17 00:00:00 2001 From: Amir Sarabadani Date: Sat, 29 Aug 2020 23:51:34 +0200 Subject: [PATCH] Make CX and Analytics patches too --- gerrit.py | 2 +- new_wikis_handler.py | 37 ++++++++++++++++++++----- patch_makers.py | 66 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 95 insertions(+), 10 deletions(-) diff --git a/gerrit.py b/gerrit.py index b6626f2..39d63ee 100644 --- a/gerrit.py +++ b/gerrit.py @@ -80,6 +80,7 @@ class ShellMixin: self.check_call(['git', 'config', 'user.name', creds['name']]) self.check_call(['git', 'config', 'user.email', creds['email']]) self.check_call(['git', 'submodule', 'update', '--init']) + load_ssh_key() self.check_call(['scp', '-p', '-P', '29418', creds['name'] + '@gerrit.wikimedia.org:hooks/commit-msg', '.git/hooks/']) @@ -126,7 +127,6 @@ class GerritBot(ShellMixin): with open('.git/COMMIT_EDITMSG', 'w') as f: f.write(self.commit_message) self.check_call(['git', 'commit', '-F', '.git/COMMIT_EDITMSG']) - load_ssh_key() self.check_call(self.build_push_command( {'hashtags': ['automated-wiki-creation'], 'repo': self.name})) diff --git a/new_wikis_handler.py b/new_wikis_handler.py index 789d43d..2ed132a 100644 --- a/new_wikis_handler.py +++ b/new_wikis_handler.py @@ -6,7 +6,8 @@ import socket import requests from lib import Client -from patch_makers import DnsPatchMaker, WikimediaMessagesPatchMaker +from patch_makers import (AnalyticsPatchMaker, CxPatchMaker, DnsPatchMaker, + WikimediaMessagesPatchMaker) final_text = '' gerrit_path = 'https://gerrit.wikimedia.org/g/' @@ -62,17 +63,28 @@ def handle_restbase(url): add_checklist(gerrit_path + path, 'RESTbase', url in restbase) -def handle_cx(language_code): +def handle_cx(language_code, bug_id): path = get_gerrit_path( 'mediawiki/services/cxserver', 'config/languages.yaml' ) cxconfig = get_file_from_gerrit(path) - add_checklist(gerrit_path + path, 'CX Config', - '\n- ' + language_code in cxconfig) + cx = '\n- ' + language_code in cxconfig + add_checklist(gerrit_path + path, 'CX Config', cx) + if cx: + return + + r = requests.get( + 'https://gerrit.wikimedia.org/r/changes/' + '?q=bug:{}+project:mediawiki/services/cxserver'.format(bug_id)) + b = json.loads('\n'.join(r.text.split('\n')[1:])) + if b: + return + maker = CxPatchMaker(lang, bug_id) + maker.run() -def handle_analytics(url): +def handle_analytics(url, bug_id): path = get_gerrit_path( 'analytics/refinery', 'static_data/pageview/whitelist/whitelist.tsv' @@ -80,6 +92,17 @@ def handle_analytics(url): refinery_whitelist = get_file_from_gerrit(path) add_checklist(gerrit_path + path, 'Analytics refinery', url in refinery_whitelist) + if url in refinery_whitelist: + return + + r = requests.get( + 'https://gerrit.wikimedia.org/r/changes/' + '?q=bug:{}+project:analytics/refinery'.format(bug_id)) + b = json.loads('\n'.join(r.text.split('\n')[1:])) + if b: + return + maker = AnalyticsPatchMaker(lang, bug_id) + maker.run() def handle_pywikibot(family, language_code): @@ -413,8 +436,8 @@ def hande_task(task_details): add_text('\n-------\n**Post install automatic checklist:**') handle_restbase(url) - handle_cx(language_code) - handle_analytics('.'.join(parts[:2])) + handle_cx(language_code, task_tid) + handle_analytics('.'.join(parts[:2]), task_tid) handle_pywikibot(parts[1], language_code) handle_wikidata(db_name) add_text(' [] Import from Incubator') diff --git a/patch_makers.py b/patch_makers.py index 55ebe00..1fd2c83 100644 --- a/patch_makers.py +++ b/patch_makers.py @@ -1,4 +1,5 @@ import json +from datetime import date from gerrit import GerritBot @@ -54,7 +55,7 @@ class WikimediaMessagesPatchMaker(GerritBot): indent='\t', sort_keys=True)) -def DnsPatchMaker(): +class DnsPatchMaker(GerritBot): def __init__(self, lang, bug_id): self.wiki_lang = lang super().__init__( @@ -79,4 +80,65 @@ def DnsPatchMaker(): langs.append(" '{}',".format(self.wiki_lang)) langs.sort() with open('templates/helpers/langlist.tmpl', 'w') as f: - f.write('\n'.join(footer) + '\n'.join(langs) + '\n'.join(footer)) + f.write('\n'.join(header) + '\n' + + '\n'.join(langs) + '\n' + '\n'.join(footer)) + + +class CxPatchMaker(GerritBot): + def __init__(self, lang, bug_id): + self.wiki_lang = lang + super().__init__( + 'mediawiki/services/cxserver', + 'Add {} to languages \n\nBug:{}'.format(lang, bug_id) + ) + + def changes(self): + with open('config/languages.yaml', 'r') as f: + lines = f.read().split('\n')[:-1] + lines.append("- {}".format(self.wiki_lang)) + lines.sort() + with open('config/languages.yaml', 'w') as f: + f.write('\n'.join(lines) + '\n') + + +class AnalyticsPatchMaker(GerritBot): + def __init__(self, project, bug_id): + self.project = project + super().__init__( + 'analytics/refinery', + 'Add {} to pageview whitelist \n\nBug:{}'.format(project, bug_id) + ) + + def changes(self): + with open('static_data/pageview/whitelist/whitelist.tsv', 'r') as f: + lines = f.read().split('\n') + projects = [] + non_projects = [] + for line in lines: + if line.startswith('project'): + projects.append(line) + else: + non_projects.append(line) + today = date.today() + projects.append('project\t{}\t{}'.format( + self.project, + today.strftime("%Y-%m-%d 00:00:00") + )) + projects.sort() + with open('static_data/pageview/whitelist/whitelist.tsv', 'w') as f: + f.write('\n'.join(projects) + '\n' + '\n'.join(non_projects)) + + +class CxPatchMakerTemp(GerritBot): + def __init__(self): + super().__init__( + 'mediawiki/services/cxserver', + 'Order entries by alphabetical order\n\nThis would make creating automated patches easier\n\nBug: T253439' + ) + + def changes(self): + with open('config/languages.yaml', 'r') as f: + lines = f.read().split('\n')[:-1] + lines.sort() + with open('config/languages.yaml', 'w') as f: + f.write('\n'.join(lines) + '\n')