mirror of
https://gitlab.wikimedia.org/ladsgroup/Phabricator-maintenance-bot
synced 2024-09-18 21:28:51 +02:00
600ea016df
this can cause serious issues if unnoticed, making the code to allow for passing visibility, I'll fix the template etc. and make the code to fetch the visibility from task soon.
460 lines
14 KiB
Python
460 lines
14 KiB
Python
import base64
|
|
import json
|
|
import re
|
|
import socket
|
|
|
|
import requests
|
|
|
|
from lib import Client
|
|
from patch_makers import (AnalyticsPatchMaker, CxPatchMaker, DnsPatchMaker,
|
|
WikimediaMessagesPatchMaker)
|
|
|
|
final_text = ''
|
|
gerrit_path = 'https://gerrit.wikimedia.org/g/'
|
|
client = Client.newFromCreds()
|
|
|
|
|
|
def add_text(a):
|
|
global final_text
|
|
final_text += a + '\n'
|
|
|
|
|
|
def add_checklist(url, text, checked):
|
|
if checked:
|
|
add_text(' [x] [[{}|{}]]'.format(url, text))
|
|
else:
|
|
add_text(' [] [[{}|{}]]'.format(url, text))
|
|
|
|
|
|
def get_file_from_gerrit(path):
|
|
gerrit_url = 'https://gerrit.wikimedia.org/g/'
|
|
url = gerrit_url + '{0}?format=TEXT'.format(path)
|
|
r = requests.get(url)
|
|
if r.status_code == 200:
|
|
return base64.b64decode(r.text).decode('utf-8')
|
|
else:
|
|
return ''
|
|
|
|
|
|
def get_gerrit_path(repo, filename):
|
|
return repo + '/+/master/' + filename
|
|
|
|
|
|
def get_github_url(repo, filename):
|
|
return 'https://raw.githubusercontent.com/wikimedia/{}/master/{}'.format(
|
|
repo, filename
|
|
)
|
|
|
|
|
|
def hostname_resolves(hostname):
|
|
try:
|
|
socket.gethostbyname(hostname)
|
|
except socket.error:
|
|
return False
|
|
return True
|
|
|
|
|
|
def handle_restbase(url):
|
|
path = get_gerrit_path(
|
|
'mediawiki/services/restbase/deploy',
|
|
'scap/vars.yaml'
|
|
)
|
|
restbase = get_file_from_gerrit(path)
|
|
add_checklist(gerrit_path + path, 'RESTbase', url in restbase)
|
|
|
|
|
|
def handle_cx(language_code, bug_id):
|
|
path = get_gerrit_path(
|
|
'mediawiki/services/cxserver',
|
|
'config/languages.yaml'
|
|
)
|
|
cxconfig = get_file_from_gerrit(path)
|
|
cx = '\n- ' + language_code in cxconfig
|
|
add_checklist(gerrit_path + path, 'CX Config', cx)
|
|
if cx:
|
|
return
|
|
|
|
r = requests.get(
|
|
'https://gerrit.wikimedia.org/r/changes/'
|
|
'?q=bug:{}+project:mediawiki/services/cxserver'.format(bug_id))
|
|
b = json.loads('\n'.join(r.text.split('\n')[1:]))
|
|
if b:
|
|
return
|
|
maker = CxPatchMaker(lang, bug_id)
|
|
maker.run()
|
|
|
|
|
|
def handle_analytics(url, bug_id):
|
|
return
|
|
path = get_gerrit_path(
|
|
'analytics/refinery',
|
|
'static_data/pageview/whitelist/whitelist.tsv'
|
|
)
|
|
refinery_whitelist = get_file_from_gerrit(path)
|
|
add_checklist(gerrit_path + path, 'Analytics refinery',
|
|
url in refinery_whitelist)
|
|
if url in refinery_whitelist:
|
|
return
|
|
|
|
r = requests.get(
|
|
'https://gerrit.wikimedia.org/r/changes/'
|
|
'?q=bug:{}+project:analytics/refinery'.format(bug_id))
|
|
b = json.loads('\n'.join(r.text.split('\n')[1:]))
|
|
if b:
|
|
return
|
|
maker = AnalyticsPatchMaker(url, bug_id)
|
|
maker.run()
|
|
|
|
|
|
def handle_pywikibot(family, language_code):
|
|
path = get_gerrit_path(
|
|
'pywikibot/core',
|
|
'pywikibot/families/{}_family.py'.format(family)
|
|
)
|
|
pywikibot = get_file_from_gerrit(path)
|
|
add_checklist(gerrit_path + path, 'Pywikibot',
|
|
"'{}'".format(language_code) in pywikibot)
|
|
|
|
|
|
def handle_wikidata(db_name):
|
|
url = 'https://www.wikidata.org/w/api.php'
|
|
wikiata_help_page = requests.get(url, params={
|
|
'action': 'help',
|
|
'modules': 'wbgetentities'
|
|
}).text
|
|
add_checklist(url, 'Wikidata', db_name in wikiata_help_page)
|
|
|
|
|
|
def handle_special_wiki_apache(parts):
|
|
file_path = 'modules/mediawiki/manifests/web/prod_sites.pp'
|
|
apache_file = get_file_from_gerrit(
|
|
'operations/puppet/+/production/' + file_path)
|
|
url = '.'.join(parts)
|
|
return url in apache_file
|
|
|
|
|
|
def post_a_comment(comment):
|
|
comment = 'Hello, I am helping on creating this wiki. ' + comment + \
|
|
' ^_^ Sincerely, your Fully Automated Resource Tackler'
|
|
pass
|
|
|
|
|
|
def handle_subticket_for_cloud(task_details, db_name, wiki_status="unknown"):
|
|
hasSubtasks = client.getTaskSubtasks(task_details['phid'])
|
|
if hasSubtasks:
|
|
return
|
|
|
|
client.createSubtask("The new wiki's visibility will be: **%s**." % wiki_status, [
|
|
'PHID-PROJ-hwibeuyzizzy4xzunfsk', # DBA
|
|
'PHID-PROJ-bj6y6ks7ampcwcignhce' # Data services
|
|
], task_details['phid'], 'Prepare and check storage layer for ' + db_name)
|
|
|
|
|
|
def get_dummy_wiki(shard, family):
|
|
if family == "wiktionary":
|
|
return {
|
|
"s3": "aawiki",
|
|
"s5": "mhwiktionary",
|
|
}.get(shard, "?????")
|
|
else:
|
|
return {
|
|
"s3": "aawiki",
|
|
"s5": "muswiki"
|
|
}.get(shard, "?????")
|
|
|
|
|
|
def create_patch_for_wikimedia_messages(
|
|
db_name, english_name, url, lang, bug_id):
|
|
if not english_name:
|
|
return
|
|
r = requests.get(
|
|
'https://gerrit.wikimedia.org/r/changes/?q='
|
|
'bug:{}+project:mediawiki/extensions/WikimediaMessages'.format(bug_id))
|
|
b = json.loads('\n'.join(r.text.split('\n')[1:]))
|
|
if b:
|
|
return
|
|
maker = WikimediaMessagesPatchMaker(
|
|
db_name, english_name, url, lang, bug_id)
|
|
maker.run()
|
|
|
|
|
|
def handle_dns(special, url, language_code, task_tid):
|
|
dns_path = get_gerrit_path(
|
|
'operations/dns',
|
|
'templates/wikimedia.org' if special else
|
|
'templates/helpers/langlist.tmpl')
|
|
dns_url = gerrit_path + dns_path
|
|
dns = hostname_resolves(url)
|
|
if not dns:
|
|
if not special:
|
|
create_patch_for_dns(language_code, task_tid)
|
|
add_checklist(dns_url, 'DNS', dns)
|
|
return dns
|
|
|
|
|
|
def handle_apache(special, parts):
|
|
if not special:
|
|
add_text(' [x] Apache config (Not needed)')
|
|
return True
|
|
|
|
file_path = 'modules/mediawiki/manifests/web/prod_sites.pp'
|
|
apache_url = gerrit_path + \
|
|
'operations/puppet/+/production/' + file_path
|
|
if not handle_special_wiki_apache(parts):
|
|
apache = False
|
|
else:
|
|
apache = True
|
|
add_checklist(apache_url, 'Apache config', apache)
|
|
return apache
|
|
|
|
|
|
def handle_langdb(language_code):
|
|
langdb_url = get_github_url('language-data', 'data/langdb.yaml')
|
|
r = requests.get(langdb_url)
|
|
config = 'Language configuration in language data repo'
|
|
if re.search(r'\n *?' + language_code + ':', r.text):
|
|
langdb = True
|
|
else:
|
|
langdb = False
|
|
add_checklist(langdb_url, config, langdb)
|
|
return langdb
|
|
|
|
|
|
def handle_wikimedia_messages_one(
|
|
db_name,
|
|
wiki_spec,
|
|
url,
|
|
language_code,
|
|
task_tid):
|
|
path = get_gerrit_path(
|
|
'mediawiki/extensions/WikimediaMessages',
|
|
'i18n/wikimediaprojectnames/en.json'
|
|
)
|
|
wikimedia_messages_data = get_file_from_gerrit(path)
|
|
wikimedia_messages_data = json.loads(wikimedia_messages_data)
|
|
if 'project-localized-name-' + db_name in wikimedia_messages_data:
|
|
wikimedia_messages_one = True
|
|
else:
|
|
wikimedia_messages_one = False
|
|
english_name = wiki_spec.get('Project name (English)')
|
|
create_patch_for_wikimedia_messages(
|
|
db_name, english_name, url, language_code, task_tid)
|
|
add_checklist(gerrit_path + path,
|
|
'Wikimedia messages configuration', wikimedia_messages_one)
|
|
url = 'https://en.wikipedia.org/wiki/' + \
|
|
'MediaWiki:Project-localized-name-' + db_name
|
|
r = requests.get(url)
|
|
if 'Wikipedia does not have a' not in r.text:
|
|
wikimedia_messages_one_deployed = True
|
|
add_text(' [x] [[{}|deployed]]'.format(url))
|
|
else:
|
|
wikimedia_messages_one_deployed = False
|
|
add_text(' [] [[{}|deployed]]'.format(url))
|
|
|
|
return wikimedia_messages_one and wikimedia_messages_one_deployed
|
|
|
|
|
|
def handle_wikimedia_messages_two(db_name, parts):
|
|
config = 'Wikimedia messages (interwiki search result) configuration'
|
|
if parts[1] != 'wikipedia':
|
|
add_text(' [x] {} (not needed)'.format(config))
|
|
return True
|
|
|
|
path = get_gerrit_path(
|
|
'mediawiki/extensions/WikimediaMessages',
|
|
'i18n/wikimediainterwikisearchresults/en.json'
|
|
)
|
|
search_messages_data = json.loads(get_file_from_gerrit(path))
|
|
if 'search-interwiki-results-' + db_name in search_messages_data:
|
|
wikimedia_messages_two = True
|
|
else:
|
|
wikimedia_messages_two = False
|
|
add_checklist(
|
|
gerrit_path + path,
|
|
config,
|
|
wikimedia_messages_two)
|
|
url = 'https://en.wikipedia.org/wiki/' + \
|
|
'MediaWiki:Search-interwiki-results-' + db_name
|
|
r = requests.get(url)
|
|
if 'Wikipedia does not have a' not in r.text:
|
|
wikimedia_messages_two_deployed = True
|
|
add_text(' [x] [[{}|deployed]]'.format(url))
|
|
else:
|
|
wikimedia_messages_two_deployed = False
|
|
add_text(' [] [[{}|deployed]]'.format(url))
|
|
return wikimedia_messages_two and wikimedia_messages_two_deployed
|
|
|
|
|
|
def create_patch_for_dns(lang, bug_id):
|
|
r = requests.get(
|
|
'https://gerrit.wikimedia.org/r/changes/'
|
|
'?q=bug:{}+project:operations/dns'.format(bug_id))
|
|
b = json.loads('\n'.join(r.text.split('\n')[1:]))
|
|
if b:
|
|
return
|
|
maker = DnsPatchMaker(lang, bug_id)
|
|
maker.run()
|
|
|
|
|
|
def handle_core_lang(language_code):
|
|
core_messages_url = get_github_url(
|
|
'mediawiki',
|
|
'languages/messages/Messages{}.php'.format(
|
|
language_code[0].upper() + language_code[1:]))
|
|
r = requests.get(core_messages_url)
|
|
if r.status_code == 200:
|
|
core_lang = True
|
|
else:
|
|
core_lang = False
|
|
add_checklist(core_messages_url,
|
|
'Language configuration in mediawiki core', core_lang)
|
|
return core_lang
|
|
|
|
|
|
def get_db_name(wiki_spec, parts):
|
|
db_name = wiki_spec.get('Database name')
|
|
if not db_name:
|
|
if parts[1] == 'wikipedia':
|
|
db_name = parts[0].replace('-', '_') + 'wiki'
|
|
else:
|
|
db_name = parts[0].replace('-', '_') + parts[1]
|
|
return db_name
|
|
|
|
|
|
def sync_file(path, summary):
|
|
return '`scap sync-file {} "{}"`'.format(path, summary)
|
|
|
|
|
|
def add_create_instructions(parts, shard, language_code, db_name, task_tid):
|
|
add_text('\n-------')
|
|
add_text('**Step by step commands**:')
|
|
dummy_wiki = get_dummy_wiki(shard, parts[1])
|
|
add_text('On deploy1001:')
|
|
add_text('`cd /srv/mediawiki-staging/`')
|
|
add_text('`git fetch`')
|
|
add_text('`git log -p HEAD..@{u}`')
|
|
add_text('`git rebase`')
|
|
add_text('On mwmaint1002:')
|
|
add_text('`scap pull`')
|
|
addwiki_path = 'mwscript extensions/WikimediaMaintenance/addWiki.php'
|
|
add_text(
|
|
'`{addwiki_path} --wiki={dummy} {lang} {family} {db} {url}`'.format(
|
|
addwiki_path=addwiki_path,
|
|
dummy=dummy_wiki,
|
|
lang=language_code,
|
|
family=parts[1],
|
|
db=db_name,
|
|
url='.'.join(parts)))
|
|
summary = 'Creating {db_name} ({phab})'.format(
|
|
db_name=db_name, phab=task_tid)
|
|
add_text('On deploy1001:')
|
|
if shard != "s3":
|
|
add_text(sync_file('wmf-config/db-eqiad.php', summary))
|
|
add_text(sync_file('wmf-config/db-codfw.php', summary))
|
|
add_text(sync_file('dblists', summary))
|
|
add_text('`scap sync-wikiversions "{}"`'.format(summary))
|
|
if parts[1] == 'wikimedia':
|
|
add_text(sync_file('multiversion/MWMultiVersion.php', summary))
|
|
add_text(sync_file('static/images/project-logos/', summary))
|
|
add_text(sync_file('wmf-config/InitialiseSettings.php', summary))
|
|
if parts[1] != 'wikimedia':
|
|
add_text(sync_file('langlist', summary))
|
|
add_text('`scap update-interwiki-cache`')
|
|
|
|
|
|
def update_task_report(task_details):
|
|
global final_text
|
|
if not final_text:
|
|
return
|
|
old_report = re.findall(
|
|
r'(\n\n------\n\*\*Pre-install automatic checklist:'
|
|
r'\*\*.+?\n\*\*End of automatic output\*\*\n)',
|
|
task_details['description'], re.DOTALL)
|
|
if not old_report:
|
|
print('old report not found, appending')
|
|
client.setTaskDescription(
|
|
task_details['phid'], task_details['description'] + final_text)
|
|
else:
|
|
if old_report[0] != final_text:
|
|
print('Updating old report')
|
|
client.setTaskDescription(
|
|
task_details['phid'],
|
|
task_details['description'].replace(
|
|
old_report[0],
|
|
final_text))
|
|
|
|
|
|
def hande_task(task_details):
|
|
global final_text
|
|
final_text = ''
|
|
print('Checking T%s' % task_details['id'])
|
|
task_tid = 'T' + task_details['id']
|
|
|
|
# Extract wiki config
|
|
wiki_spec = {}
|
|
for case in re.findall(
|
|
r'\n- *?\*\*(.+?):\*\* *?(.+)',
|
|
task_details['description']):
|
|
wiki_spec[case[0].strip()] = case[1].strip()
|
|
language_code = wiki_spec.get('Language code')
|
|
if not language_code:
|
|
print('lang code not found, skipping')
|
|
return
|
|
url = wiki_spec.get('Site URL')
|
|
if not url:
|
|
print('url not found, skipping')
|
|
return
|
|
parts = url.split('.')
|
|
if len(parts) != 3 or parts[2] != 'org':
|
|
print('the url looks weird, skipping')
|
|
return
|
|
db_name = get_db_name(wiki_spec, parts)
|
|
shard = wiki_spec.get('Shard', 'TBD')
|
|
shardDecided = shard != "TBD"
|
|
special = parts[1] == 'wikimedia'
|
|
|
|
add_text('\n\n------\n**Pre-install automatic checklist:**')
|
|
if shardDecided:
|
|
add_text(' [X] #DBA decided about the shard')
|
|
else:
|
|
add_text(' [] #DBA decided about the shard')
|
|
dns = handle_dns(special, url, language_code, task_tid)
|
|
if not special and wiki_spec.get('Special', '').lower() != 'yes':
|
|
handle_subticket_for_cloud(task_details, db_name)
|
|
apache = handle_apache(special, parts)
|
|
langdb = handle_langdb(language_code)
|
|
core_lang = handle_core_lang(language_code)
|
|
wm_message_one = handle_wikimedia_messages_one(
|
|
db_name, wiki_spec, url, language_code, task_tid
|
|
)
|
|
wm_message_two = handle_wikimedia_messages_two(db_name, parts)
|
|
|
|
if dns and apache and langdb and core_lang and wm_message_one and \
|
|
wm_message_two and shardDecided:
|
|
add_text('**The Wiki is ready to be created.**')
|
|
else:
|
|
add_text('**The creation is blocked until these part are all done.**')
|
|
|
|
add_text('\n-------\n**Post install automatic checklist:**')
|
|
handle_restbase(url)
|
|
handle_cx(language_code, task_tid)
|
|
handle_analytics('.'.join(parts[:2]), task_tid)
|
|
handle_pywikibot(parts[1], language_code)
|
|
handle_wikidata(db_name)
|
|
add_text(' [] Import from Incubator')
|
|
add_text(' [] Clean up old interwiki links')
|
|
add_create_instructions(parts, shard, language_code, db_name, task_tid)
|
|
add_text('\n**End of automatic output**')
|
|
|
|
|
|
def main():
|
|
open_create_wikis_phid = 'PHID-PROJ-kmpu7gznmc2edea3qn2x'
|
|
for phid in client.getTasksWithProject(
|
|
open_create_wikis_phid, statuses=['open']):
|
|
task_details = client.taskDetails(phid)
|
|
hande_task(task_details)
|
|
update_task_report(task_details)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|