mirror of
https://gitlab.wikimedia.org/ladsgroup/Phabricator-maintenance-bot
synced 2024-11-08 13:12:38 +01:00
460 lines
14 KiB
Python
460 lines
14 KiB
Python
import base64
|
|
import json
|
|
import re
|
|
import socket
|
|
|
|
import requests
|
|
|
|
from lib import Client
|
|
from patch_makers import (AnalyticsPatchMaker, CxPatchMaker, DnsPatchMaker,
|
|
WikimediaMessagesPatchMaker)
|
|
|
|
final_text = ''
|
|
gerrit_path = 'https://gerrit.wikimedia.org/g/'
|
|
client = Client.newFromCreds()
|
|
|
|
|
|
def add_text(a):
|
|
global final_text
|
|
final_text += a + '\n'
|
|
|
|
|
|
def add_checklist(url, text, checked):
|
|
if checked:
|
|
add_text(' [x] [[{}|{}]]'.format(url, text))
|
|
else:
|
|
add_text(' [] [[{}|{}]]'.format(url, text))
|
|
|
|
|
|
def get_file_from_gerrit(path):
|
|
gerrit_url = 'https://gerrit.wikimedia.org/g/'
|
|
url = gerrit_url + '{0}?format=TEXT'.format(path)
|
|
r = requests.get(url)
|
|
if r.status_code == 200:
|
|
return base64.b64decode(r.text).decode('utf-8')
|
|
else:
|
|
return ''
|
|
|
|
|
|
def get_gerrit_path(repo, filename):
|
|
return repo + '/+/master/' + filename
|
|
|
|
|
|
def get_github_url(repo, filename):
|
|
return 'https://raw.githubusercontent.com/wikimedia/{}/master/{}'.format(
|
|
repo, filename
|
|
)
|
|
|
|
|
|
def hostname_resolves(hostname):
|
|
try:
|
|
socket.gethostbyname(hostname)
|
|
except socket.error:
|
|
return False
|
|
return True
|
|
|
|
|
|
def handle_restbase(url):
|
|
path = get_gerrit_path(
|
|
'mediawiki/services/restbase/deploy',
|
|
'scap/vars.yaml'
|
|
)
|
|
restbase = get_file_from_gerrit(path)
|
|
add_checklist(gerrit_path + path, 'RESTbase', url in restbase)
|
|
|
|
|
|
def handle_cx(language_code, bug_id):
|
|
path = get_gerrit_path(
|
|
'mediawiki/services/cxserver',
|
|
'config/languages.yaml'
|
|
)
|
|
cxconfig = get_file_from_gerrit(path)
|
|
cx = '\n- ' + language_code in cxconfig
|
|
add_checklist(gerrit_path + path, 'CX Config', cx)
|
|
if cx:
|
|
return
|
|
|
|
r = requests.get(
|
|
'https://gerrit.wikimedia.org/r/changes/'
|
|
'?q=bug:{}+project:mediawiki/services/cxserver'.format(bug_id))
|
|
b = json.loads('\n'.join(r.text.split('\n')[1:]))
|
|
if b:
|
|
return
|
|
maker = CxPatchMaker(lang, bug_id)
|
|
maker.run()
|
|
|
|
|
|
def handle_analytics(url, bug_id):
|
|
return
|
|
path = get_gerrit_path(
|
|
'analytics/refinery',
|
|
'static_data/pageview/whitelist/whitelist.tsv'
|
|
)
|
|
refinery_whitelist = get_file_from_gerrit(path)
|
|
add_checklist(gerrit_path + path, 'Analytics refinery',
|
|
url in refinery_whitelist)
|
|
if url in refinery_whitelist:
|
|
return
|
|
|
|
r = requests.get(
|
|
'https://gerrit.wikimedia.org/r/changes/'
|
|
'?q=bug:{}+project:analytics/refinery'.format(bug_id))
|
|
b = json.loads('\n'.join(r.text.split('\n')[1:]))
|
|
if b:
|
|
return
|
|
maker = AnalyticsPatchMaker(url, bug_id)
|
|
maker.run()
|
|
|
|
|
|
def handle_pywikibot(family, language_code):
|
|
path = get_gerrit_path(
|
|
'pywikibot/core',
|
|
'pywikibot/families/{}_family.py'.format(family)
|
|
)
|
|
pywikibot = get_file_from_gerrit(path)
|
|
add_checklist(gerrit_path + path, 'Pywikibot',
|
|
"'{}'".format(language_code) in pywikibot)
|
|
|
|
|
|
def handle_wikidata(db_name):
|
|
url = 'https://www.wikidata.org/w/api.php'
|
|
wikiata_help_page = requests.get(url, params={
|
|
'action': 'help',
|
|
'modules': 'wbgetentities'
|
|
}).text
|
|
add_checklist(url, 'Wikidata', db_name in wikiata_help_page)
|
|
|
|
|
|
def handle_special_wiki_apache(parts):
|
|
file_path = 'modules/mediawiki/manifests/web/prod_sites.pp'
|
|
apache_file = get_file_from_gerrit(
|
|
'operations/puppet/+/production/' + file_path)
|
|
url = '.'.join(parts)
|
|
return url in apache_file
|
|
|
|
|
|
def post_a_comment(comment):
|
|
comment = 'Hello, I am helping on creating this wiki. ' + comment + \
|
|
' ^_^ Sincerely, your Fully Automated Resource Tackler'
|
|
pass
|
|
|
|
|
|
def handle_subticket_for_cloud(task_details, db_name):
|
|
hasSubtasks = client.getTaskSubtasks(task_details['phid'])
|
|
if hasSubtasks:
|
|
return
|
|
|
|
client.createSubtask('The new wiki is going to be **public**.', [
|
|
'PHID-PROJ-hwibeuyzizzy4xzunfsk', # DBA
|
|
'PHID-PROJ-bj6y6ks7ampcwcignhce' # Data services
|
|
], task_details['phid'], 'Prepare and check storage layer for ' + db_name)
|
|
|
|
|
|
def get_dummy_wiki(shard, family):
|
|
if family == "wiktionary":
|
|
return {
|
|
"s3": "aawiki",
|
|
"s5": "mhwiktionary",
|
|
}.get(shard, "?????")
|
|
else:
|
|
return {
|
|
"s3": "aawiki",
|
|
"s5": "muswiki"
|
|
}.get(shard, "?????")
|
|
|
|
|
|
def create_patch_for_wikimedia_messages(
|
|
db_name, english_name, url, lang, bug_id):
|
|
if not english_name:
|
|
return
|
|
r = requests.get(
|
|
'https://gerrit.wikimedia.org/r/changes/?q='
|
|
'bug:{}+project:mediawiki/extensions/WikimediaMessages'.format(bug_id))
|
|
b = json.loads('\n'.join(r.text.split('\n')[1:]))
|
|
if b:
|
|
return
|
|
maker = WikimediaMessagesPatchMaker(
|
|
db_name, english_name, url, lang, bug_id)
|
|
maker.run()
|
|
|
|
|
|
def handle_dns(special, url, language_code, task_tid):
|
|
dns_path = get_gerrit_path(
|
|
'operations/dns',
|
|
'templates/wikimedia.org' if special else
|
|
'templates/helpers/langlist.tmpl')
|
|
dns_url = gerrit_path + dns_path
|
|
dns = hostname_resolves(url)
|
|
if not dns:
|
|
if not special:
|
|
create_patch_for_dns(language_code, task_tid)
|
|
add_checklist(dns_url, 'DNS', dns)
|
|
return dns
|
|
|
|
|
|
def handle_apache(special, parts):
|
|
if not special:
|
|
add_text(' [x] Apache config (Not needed)')
|
|
return True
|
|
|
|
file_path = 'modules/mediawiki/manifests/web/prod_sites.pp'
|
|
apache_url = gerrit_path + \
|
|
'operations/puppet/+/production/' + file_path
|
|
if not handle_special_wiki_apache(parts):
|
|
apache = False
|
|
else:
|
|
apache = True
|
|
add_checklist(apache_url, 'Apache config', apache)
|
|
return apache
|
|
|
|
|
|
def handle_langdb(language_code):
|
|
langdb_url = get_github_url('language-data', 'data/langdb.yaml')
|
|
r = requests.get(langdb_url)
|
|
config = 'Language configuration in language data repo'
|
|
if re.search(r'\n *?' + language_code + ':', r.text):
|
|
langdb = True
|
|
else:
|
|
langdb = False
|
|
add_checklist(langdb_url, config, langdb)
|
|
return langdb
|
|
|
|
|
|
def handle_wikimedia_messages_one(
|
|
db_name,
|
|
wiki_spec,
|
|
url,
|
|
language_code,
|
|
task_tid):
|
|
path = get_gerrit_path(
|
|
'mediawiki/extensions/WikimediaMessages',
|
|
'i18n/wikimediaprojectnames/en.json'
|
|
)
|
|
wikimedia_messages_data = get_file_from_gerrit(path)
|
|
wikimedia_messages_data = json.loads(wikimedia_messages_data)
|
|
if 'project-localized-name-' + db_name in wikimedia_messages_data:
|
|
wikimedia_messages_one = True
|
|
else:
|
|
wikimedia_messages_one = False
|
|
english_name = wiki_spec.get('Project name (English)')
|
|
create_patch_for_wikimedia_messages(
|
|
db_name, english_name, url, language_code, task_tid)
|
|
add_checklist(gerrit_path + path,
|
|
'Wikimedia messages configuration', wikimedia_messages_one)
|
|
url = 'https://en.wikipedia.org/wiki/' + \
|
|
'MediaWiki:Project-localized-name-' + db_name
|
|
r = requests.get(url)
|
|
if 'Wikipedia does not have a' not in r.text:
|
|
wikimedia_messages_one_deployed = True
|
|
add_text(' [x] [[{}|deployed]]'.format(url))
|
|
else:
|
|
wikimedia_messages_one_deployed = False
|
|
add_text(' [] [[{}|deployed]]'.format(url))
|
|
|
|
return wikimedia_messages_one and wikimedia_messages_one_deployed
|
|
|
|
|
|
def handle_wikimedia_messages_two(db_name, parts):
|
|
config = 'Wikimedia messages (interwiki search result) configuration'
|
|
if parts[1] != 'wikipedia':
|
|
add_text(' [x] {} (not needed)'.format(config))
|
|
return True
|
|
|
|
path = get_gerrit_path(
|
|
'mediawiki/extensions/WikimediaMessages',
|
|
'i18n/wikimediainterwikisearchresults/en.json'
|
|
)
|
|
search_messages_data = json.loads(get_file_from_gerrit(path))
|
|
if 'search-interwiki-results-' + db_name in search_messages_data:
|
|
wikimedia_messages_two = True
|
|
else:
|
|
wikimedia_messages_two = False
|
|
add_checklist(
|
|
gerrit_path + path,
|
|
config,
|
|
wikimedia_messages_two)
|
|
url = 'https://en.wikipedia.org/wiki/' + \
|
|
'MediaWiki:Search-interwiki-results-' + db_name
|
|
r = requests.get(url)
|
|
if 'Wikipedia does not have a' not in r.text:
|
|
wikimedia_messages_two_deployed = True
|
|
add_text(' [x] [[{}|deployed]]'.format(url))
|
|
else:
|
|
wikimedia_messages_two_deployed = False
|
|
add_text(' [] [[{}|deployed]]'.format(url))
|
|
return wikimedia_messages_two and wikimedia_messages_two_deployed
|
|
|
|
|
|
def create_patch_for_dns(lang, bug_id):
|
|
r = requests.get(
|
|
'https://gerrit.wikimedia.org/r/changes/'
|
|
'?q=bug:{}+project:operations/dns'.format(bug_id))
|
|
b = json.loads('\n'.join(r.text.split('\n')[1:]))
|
|
if b:
|
|
return
|
|
maker = DnsPatchMaker(lang, bug_id)
|
|
maker.run()
|
|
|
|
|
|
def handle_core_lang(language_code):
|
|
core_messages_url = get_github_url(
|
|
'mediawiki',
|
|
'languages/messages/Messages{}.php'.format(
|
|
language_code[0].upper() + language_code[1:]))
|
|
r = requests.get(core_messages_url)
|
|
if r.status_code == 200:
|
|
core_lang = True
|
|
else:
|
|
core_lang = False
|
|
add_checklist(core_messages_url,
|
|
'Language configuration in mediawiki core', core_lang)
|
|
return core_lang
|
|
|
|
|
|
def get_db_name(wiki_spec, parts):
|
|
db_name = wiki_spec.get('Database name')
|
|
if not db_name:
|
|
if parts[1] == 'wikipedia':
|
|
db_name = parts[0].replace('-', '_') + 'wiki'
|
|
else:
|
|
db_name = parts[0].replace('-', '_') + parts[1]
|
|
return db_name
|
|
|
|
|
|
def sync_file(path, summary):
|
|
return '`scap sync-file {} "{}"`'.format(path, summary)
|
|
|
|
|
|
def add_create_instructions(parts, shard, language_code, db_name, task_tid):
|
|
add_text('\n-------')
|
|
add_text('**Step by step commands**:')
|
|
dummy_wiki = get_dummy_wiki(shard, parts[1])
|
|
add_text('On deploy1001:')
|
|
add_text('`cd /srv/mediawiki-staging/`')
|
|
add_text('`git fetch`')
|
|
add_text('`git log -p HEAD..@{u}`')
|
|
add_text('`git rebase`')
|
|
add_text('On mwmaint1002:')
|
|
add_text('`scap pull`')
|
|
addwiki_path = 'mwscript extensions/WikimediaMaintenance/addWiki.php'
|
|
add_text(
|
|
'`{addwiki_path} --wiki={dummy} {lang} {family} {db} {url}`'.format(
|
|
addwiki_path=addwiki_path,
|
|
dummy=dummy_wiki,
|
|
lang=language_code,
|
|
family=parts[1],
|
|
db=db_name,
|
|
url='.'.join(parts)))
|
|
summary = 'Creating {db_name} ({phab})'.format(
|
|
db_name=db_name, phab=task_tid)
|
|
add_text('On deploy1001:')
|
|
if shard != "s3":
|
|
add_text(sync_file('wmf-config/db-eqiad.php', summary))
|
|
add_text(sync_file('wmf-config/db-codfw.php', summary))
|
|
add_text(sync_file('dblists', summary))
|
|
add_text('`scap sync-wikiversions "{}"`'.format(summary))
|
|
if parts[1] == 'wikimedia':
|
|
add_text(sync_file('multiversion/MWMultiVersion.php', summary))
|
|
add_text(sync_file('static/images/project-logos/', summary))
|
|
add_text(sync_file('wmf-config/InitialiseSettings.php', summary))
|
|
if parts[1] != 'wikimedia':
|
|
add_text(sync_file('langlist', summary))
|
|
add_text('`scap update-interwiki-cache`')
|
|
|
|
|
|
def update_task_report(task_details):
|
|
global final_text
|
|
if not final_text:
|
|
return
|
|
old_report = re.findall(
|
|
r'(\n\n------\n\*\*Pre-install automatic checklist:'
|
|
r'\*\*.+?\n\*\*End of automatic output\*\*\n)',
|
|
task_details['description'], re.DOTALL)
|
|
if not old_report:
|
|
print('old report not found, appending')
|
|
client.setTaskDescription(
|
|
task_details['phid'], task_details['description'] + final_text)
|
|
else:
|
|
if old_report[0] != final_text:
|
|
print('Updating old report')
|
|
client.setTaskDescription(
|
|
task_details['phid'],
|
|
task_details['description'].replace(
|
|
old_report[0],
|
|
final_text))
|
|
|
|
|
|
def hande_task(task_details):
|
|
global final_text
|
|
final_text = ''
|
|
print('Checking T%s' % task_details['id'])
|
|
task_tid = 'T' + task_details['id']
|
|
|
|
# Extract wiki config
|
|
wiki_spec = {}
|
|
for case in re.findall(
|
|
r'\n- *?\*\*(.+?):\*\* *?(.+)',
|
|
task_details['description']):
|
|
wiki_spec[case[0].strip()] = case[1].strip()
|
|
language_code = wiki_spec.get('Language code')
|
|
if not language_code:
|
|
print('lang code not found, skipping')
|
|
return
|
|
url = wiki_spec.get('Site URL')
|
|
if not url:
|
|
print('url not found, skipping')
|
|
return
|
|
parts = url.split('.')
|
|
if len(parts) != 3 or parts[2] != 'org':
|
|
print('the url looks weird, skipping')
|
|
return
|
|
db_name = get_db_name(wiki_spec, parts)
|
|
shard = wiki_spec.get('Shard', 'TBD')
|
|
shardDecided = shard != "TBD"
|
|
special = parts[1] == 'wikimedia'
|
|
|
|
add_text('\n\n------\n**Pre-install automatic checklist:**')
|
|
if shardDecided:
|
|
add_text(' [X] #DBA decided about the shard')
|
|
else:
|
|
add_text(' [] #DBA decided about the shard')
|
|
dns = handle_dns(special, url, language_code, task_tid)
|
|
if not special and wiki_spec.get('Special', '').lower() != 'yes':
|
|
handle_subticket_for_cloud(task_details, db_name)
|
|
apache = handle_apache(special, parts)
|
|
langdb = handle_langdb(language_code)
|
|
core_lang = handle_core_lang(language_code)
|
|
wm_message_one = handle_wikimedia_messages_one(
|
|
db_name, wiki_spec, url, language_code, task_tid
|
|
)
|
|
wm_message_two = handle_wikimedia_messages_two(db_name, parts)
|
|
|
|
if dns and apache and langdb and core_lang and wm_message_one and \
|
|
wm_message_two and shardDecided:
|
|
add_text('**The Wiki is ready to be created.**')
|
|
else:
|
|
add_text('**The creation is blocked until these part are all done.**')
|
|
|
|
add_text('\n-------\n**Post install automatic checklist:**')
|
|
handle_restbase(url)
|
|
handle_cx(language_code, task_tid)
|
|
handle_analytics('.'.join(parts[:2]), task_tid)
|
|
handle_pywikibot(parts[1], language_code)
|
|
handle_wikidata(db_name)
|
|
add_text(' [] Import from Incubator')
|
|
add_text(' [] Clean up old interwiki links')
|
|
add_create_instructions(parts, shard, language_code, db_name, task_tid)
|
|
add_text('\n**End of automatic output**')
|
|
|
|
|
|
def main():
|
|
open_create_wikis_phid = 'PHID-PROJ-kmpu7gznmc2edea3qn2x'
|
|
for phid in client.getTasksWithProject(
|
|
open_create_wikis_phid, statuses=['open']):
|
|
task_details = client.taskDetails(phid)
|
|
hande_task(task_details)
|
|
update_task_report(task_details)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|